From c97ba0b9e6d6327a356be4d0cd6cebfab5c7aa10 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Tue, 4 Jul 2023 15:06:36 +0100 Subject: [PATCH 01/21] First pass as de-MSC-ifying space summaries - Opt for referring to it as 'room hierarchies', reflecting the route - Split msc2945.go across clientapi and roomserverapi - Still TODO fed api - Some other TODOs still to do --- clientapi/routing/room_hierarchy.go | 150 +++++ clientapi/routing/routing.go | 14 + federationapi/api/api.go | 3 +- internal/caching/cache_roomservernids.go | 1 + internal/caching/cache_space_rooms.go | 15 +- internal/caching/caches.go | 2 +- internal/caching/impl_ristretto.go | 2 +- roomserver/api/api.go | 6 + roomserver/api/query.go | 22 + roomserver/internal/api.go | 20 +- roomserver/internal/query/query.go | 2 + .../internal/query/query_room_hierarchy.go | 597 ++++++++++++++++++ roomserver/types/types.go | 35 + setup/mscs/msc2946/msc2946.go | 12 +- 14 files changed, 856 insertions(+), 25 deletions(-) create mode 100644 clientapi/routing/room_hierarchy.go create mode 100644 roomserver/internal/query/query_room_hierarchy.go diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go new file mode 100644 index 0000000000..22b29f487a --- /dev/null +++ b/clientapi/routing/room_hierarchy.go @@ -0,0 +1,150 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + "strconv" + "sync" + + "github.com/google/uuid" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" +) + +type RoomHierarchyPaginationCache struct { + cache map[string]roomserverAPI.CachedRoomHierarchyWalker + mu sync.Mutex +} + +func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache { + return RoomHierarchyPaginationCache{} +} + +func (c *RoomHierarchyPaginationCache) Get(token string) roomserverAPI.CachedRoomHierarchyWalker { + c.mu.Lock() + defer c.mu.Unlock() + line := c.cache[token] + return line +} + +func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.CachedRoomHierarchyWalker) string { + c.mu.Lock() + defer c.mu.Unlock() + token := uuid.NewString() + c.cache[token] = line + return token +} + +func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse { + parsedRoomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("room is unknown/forbidden"), + } + } + roomID := *parsedRoomID + + suggestedOnly := false // Defaults to false (spec-defined) + switch req.URL.Query().Get("suggested_only") { + case "true": + suggestedOnly = true + case "false": + case "": // Empty string is returned when query param is not set + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"), + } + } + + limit := 1000 // Default to 1000 + limitStr := req.URL.Query().Get("limit") + if limitStr != "" { + maybeLimit, err := strconv.Atoi(limitStr) + if err != nil || maybeLimit < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'limit', if set, must be a positive integer"), + } + } + limit = maybeLimit + if limit > 1000 { + limit = 1000 // Maximum limit of 1000 + } + } + + maxDepth := -1 // '-1' representing no maximum depth + maxDepthStr := req.URL.Query().Get("max_depth") + if maxDepthStr != "" { + maybeMaxDepth, err := strconv.Atoi(maxDepthStr) + if err != nil || maybeMaxDepth < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'max_depth', if set, must be a positive integer"), + } + } + maxDepth = maybeMaxDepth + } + + from := req.URL.Query().Get("from") + + var walker roomserverAPI.RoomHierarchyWalker + if from == "" { // No pagination token provided, so start new hierarchy walker + walker = rsAPI.QueryRoomHierarchy(req.Context(), types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth) + } else { // Attempt to resume cached walker + cachedWalker := paginationCache.Get(from) + + if cachedWalker == nil || !cachedWalker.ValidateParams(suggestedOnly, maxDepth) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"), + } + } + + walker = cachedWalker.GetWalker() + } + + discoveredRooms, err := walker.NextPage(limit) + + if err != nil { + // TODO + } + + nextBatch := "" + if !walker.Done() { + cacheLine := walker.GetCached() + nextBatch = paginationCache.AddLine(cacheLine) + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: MSC2946ClientResponse{ + Rooms: discoveredRooms, + NextBatch: nextBatch, + }, + } + +} + +type MSC2946ClientResponse struct { + Rooms []fclient.MSC2946Room `json:"rooms"` + NextBatch string `json:"next_batch,omitempty"` +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ab4aefddd3..dbd607eaf7 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -288,6 +288,8 @@ func Setup( // Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing! v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() + v1mux := publicAPIMux.PathPrefix("/v1/").Subrouter() + unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter() v3mux.Handle("/createRoom", @@ -505,6 +507,18 @@ func Setup( }, httputil.WithAllowGuests()), ).Methods(http.MethodPut, http.MethodOptions) + // Defined outside of handler to persist between calls + roomHierarchyPaginationCache := new(RoomHierarchyPaginationCache) + v1mux.Handle("/rooms/{roomID}/hierarchy", + httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, roomHierarchyPaginationCache) + }, httputil.WithAllowGuests()), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { if r := rateLimits.Limit(req, nil); r != nil { return *r diff --git a/federationapi/api/api.go b/federationapi/api/api.go index 756f9bc16d..6d8ae0a120 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -27,7 +27,6 @@ type FederationInternalAPI interface { QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error LookupServerKeys(ctx context.Context, s spec.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp) ([]gomatrixserverlib.ServerKeys, error) MSC2836EventRelationships(ctx context.Context, origin, dst spec.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error) - MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) // Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos. PerformBroadcastEDU( @@ -75,6 +74,8 @@ type RoomserverFederationAPI interface { GetEventAuth(ctx context.Context, origin, s spec.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error) GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error) + + MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) } type P2PFederationAPI interface { diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go index 734a3a04fa..fa0781ef3a 100644 --- a/internal/caching/cache_roomservernids.go +++ b/internal/caching/cache_roomservernids.go @@ -8,6 +8,7 @@ type RoomServerCaches interface { RoomServerNIDsCache RoomVersionCache RoomServerEventsCache + RoomHierarchyCache EventStateKeyCache EventTypeCache } diff --git a/internal/caching/cache_space_rooms.go b/internal/caching/cache_space_rooms.go index 100ab9023e..2f6d9b0266 100644 --- a/internal/caching/cache_space_rooms.go +++ b/internal/caching/cache_space_rooms.go @@ -2,15 +2,16 @@ package caching import "github.com/matrix-org/gomatrixserverlib/fclient" -type SpaceSummaryRoomsCache interface { - GetSpaceSummary(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) - StoreSpaceSummary(roomID string, r fclient.MSC2946SpacesResponse) +// RoomHierarchy cache caches responses to federated room hierarchy requests (A.K.A. 'space summaries') +type RoomHierarchyCache interface { + GetRoomHierarchy(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) + StoreRoomHierarchy(roomID string, r fclient.MSC2946SpacesResponse) } -func (c Caches) GetSpaceSummary(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) { - return c.SpaceSummaryRooms.Get(roomID) +func (c Caches) GetRoomHierarchy(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) { + return c.RoomHierarchies.Get(roomID) } -func (c Caches) StoreSpaceSummary(roomID string, r fclient.MSC2946SpacesResponse) { - c.SpaceSummaryRooms.Set(roomID, r) +func (c Caches) StoreRoomHierarchy(roomID string, r fclient.MSC2946SpacesResponse) { + c.RoomHierarchies.Set(roomID, r) } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 6bae60d59f..b5d45e8e5d 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -35,7 +35,7 @@ type Caches struct { RoomServerEventTypes Cache[types.EventTypeNID, string] // eventType NID -> eventType FederationPDUs Cache[int64, *types.HeaderedEvent] // queue NID -> PDU FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU - SpaceSummaryRooms Cache[string, fclient.MSC2946SpacesResponse] // room ID -> space response + RoomHierarchies Cache[string, fclient.MSC2946SpacesResponse] // room ID -> space response LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID } diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 00989b7601..d970cfad4b 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -147,7 +147,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm MaxAge: lesserOf(time.Hour/2, maxAge), }, }, - SpaceSummaryRooms: &RistrettoCachePartition[string, fclient.MSC2946SpacesResponse]{ // room ID -> space response + RoomHierarchies: &RistrettoCachePartition[string, fclient.MSC2946SpacesResponse]{ // room ID -> space response cache: cache, Prefix: spaceSummaryRoomsCache, Mutable: true, diff --git a/roomserver/api/api.go b/roomserver/api/api.go index c29406a1a3..c10a8bdeda 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -113,6 +113,10 @@ type QueryEventsAPI interface { QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error } +type QueryRoomHierarchyAPI interface { + QueryRoomHierarchy(ctx context.Context, caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) RoomHierarchyWalker +} + // API functions required by the syncapi type SyncRoomserverAPI interface { QueryLatestEventsAndStateAPI @@ -187,6 +191,7 @@ type ClientRoomserverAPI interface { QueryEventsAPI QuerySenderIDAPI UserRoomPrivateKeyCreator + QueryRoomHierarchyAPI QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error @@ -236,6 +241,7 @@ type FederationRoomserverAPI interface { QueryLatestEventsAndStateAPI QueryBulkStateContentAPI QuerySenderIDAPI + QueryRoomHierarchyAPI UserRoomPrivateKeyCreator AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index b6140afd56..ded27b8533 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" @@ -503,3 +504,24 @@ func (mq *MembershipQuerier) CurrentMembership(ctx context.Context, roomID spec. } return membership, err } + +type QueryRoomHierarchyRequest struct { + SuggestedOnly bool `json:"suggested_only"` + Limit int `json:"limit"` + MaxDepth int `json:"max_depth"` + From int `json:"json"` +} + +type RoomHierarchyWalker interface { + NextPage(limit int) ([]fclient.MSC2946Room, error) + Done() bool + GetCached() CachedRoomHierarchyWalker +} + +// Stripped down version of RoomHierarchyWalker suitable for caching (for pagination) +type CachedRoomHierarchyWalker interface { + // Converts this cached walker back into an actual walker, to resume walking from. + GetWalker() RoomHierarchyWalker + // Validates that the given parameters match those stored in the cache + ValidateParams(suggestedOnly bool, maxDepth int) bool +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 712c365a49..3673f0b9d2 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -91,15 +91,8 @@ func NewRoomserverAPI( NATSClient: nc, Durable: dendriteCfg.Global.JetStream.Durable("RoomserverInputConsumer"), ServerACLs: serverACLs, - Queryer: &query.Queryer{ - DB: roomserverDB, - Cache: caches, - IsLocalServerName: dendriteCfg.Global.IsLocalServerName, - ServerACLs: serverACLs, - Cfg: dendriteCfg, - }, - enableMetrics: enableMetrics, - // perform-er structs get initialised when we have a federation sender to use + enableMetrics: enableMetrics, + // perform-er structs + queryer struct get initialised when we have a federation sender to use } return a } @@ -111,6 +104,15 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio r.fsAPI = fsAPI r.KeyRing = keyRing + r.Queryer = &query.Queryer{ + DB: r.DB, + Cache: r.Cache, + IsLocalServerName: r.Cfg.Global.IsLocalServerName, + ServerACLs: r.ServerACLs, + Cfg: r.Cfg, + FSAPI: fsAPI, + } + r.Inputer = &input.Inputer{ Cfg: &r.Cfg.RoomServer, ProcessContext: r.ProcessContext, diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 39e3bd0ec6..11e5564dcf 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -32,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/api" @@ -47,6 +48,7 @@ type Queryer struct { IsLocalServerName func(spec.ServerName) bool ServerACLs *acls.ServerACLs Cfg *config.Dendrite + FSAPI fsAPI.RoomserverFederationAPI } func (r *Queryer) RestrictedRoomJoinInfo(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, localServerName spec.ServerName) (*gomatrixserverlib.RestrictedRoomJoinInfo, error) { diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go new file mode 100644 index 0000000000..e4ac75e03d --- /dev/null +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -0,0 +1,597 @@ +package query + +import ( + "context" + "encoding/json" + "sort" + "strings" + + fs "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/caching" + roomserver "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" +) + +// Query the hierarchy of a room (A.K.A 'space summary') +// +// This function returns a iterator-like struct over the hierarchy of a room. +func (r *Queryer) QueryRoomHierarchy(ctx context.Context, caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) roomserver.RoomHierarchyWalker { + walker := RoomHierarchyWalker{ + rootRoomID: roomID.String(), + caller: caller, + thisServer: r.Cfg.Global.ServerName, + rsAPI: r, + fsAPI: r.FSAPI, + ctx: ctx, + roomHierarchyCache: r.Cache, + suggestedOnly: suggestedOnly, + maxDepth: maxDepth, + unvisited: []roomVisit{{ + roomID: roomID.String(), + parentRoomID: "", + depth: 0, + }}, + } + + return &walker +} + +type stringSet map[string]struct{} + +func (s stringSet) contains(val string) bool { + _, ok := s[val] + return ok +} + +func (s stringSet) add(val string) { + s[val] = struct{}{} +} + +type RoomHierarchyWalker struct { + rootRoomID string // TODO change to spec.RoomID + caller types.DeviceOrServerName + thisServer spec.ServerName + rsAPI *Queryer + fsAPI fs.RoomserverFederationAPI + ctx context.Context + roomHierarchyCache caching.RoomHierarchyCache + suggestedOnly bool + maxDepth int + + processed stringSet + unvisited []roomVisit + + done bool +} + +const ( + ConstCreateEventContentKey = "type" + ConstCreateEventContentValueSpace = "m.space" + ConstSpaceChildEventType = "m.space.child" + ConstSpaceParentEventType = "m.space.parent" +) + +func (w *RoomHierarchyWalker) NextPage(limit int) ([]fclient.MSC2946Room, error) { + if authorised, _ := w.authorised(w.rootRoomID, ""); !authorised { + return nil, spec.Forbidden("room is unknown/forbidden") + } + + var discoveredRooms []fclient.MSC2946Room + + // Depth first -> stack data structure + for len(w.unvisited) > 0 { + if len(discoveredRooms) >= limit { + break + } + + // pop the stack + rv := w.unvisited[len(w.unvisited)-1] + w.unvisited = w.unvisited[:len(w.unvisited)-1] + // If this room has already been processed, skip. + // If this room exceeds the specified depth, skip. + if w.processed.contains(rv.roomID) || rv.roomID == "" || (w.maxDepth > 0 && rv.depth > w.maxDepth) { + continue + } + + // Mark this room as processed. + w.processed.add(rv.roomID) + + // if this room is not a space room, skip. + var roomType string + create := w.stateEvent(rv.roomID, spec.MRoomCreate, "") + if create != nil { + // escape the `.`s so gjson doesn't think it's nested + roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str + } + + // Collect rooms/events to send back (either locally or fetched via federation) + var discoveredChildEvents []fclient.MSC2946StrippedEvent + + // If we know about this room and the caller is authorised (joined/world_readable) then pull + // events locally + roomExists := w.roomExists(rv.roomID) + if !roomExists { + // attempt to query this room over federation, as either we've never heard of it before + // or we've left it and hence are not authorised (but info may be exposed regardless) + fedRes := w.federatedRoomInfo(rv.roomID, rv.vias) + if fedRes != nil { + discoveredChildEvents = fedRes.Room.ChildrenState + discoveredRooms = append(discoveredRooms, fedRes.Room) + if len(fedRes.Children) > 0 { + discoveredRooms = append(discoveredRooms, fedRes.Children...) + } + // mark this room as a space room as the federated server responded. + // we need to do this so we add the children of this room to the unvisited stack + // as these children may be rooms we do know about. + roomType = ConstCreateEventContentValueSpace + } + } else if authorised, isJoinedOrInvited := w.authorised(rv.roomID, rv.parentRoomID); authorised { + // Get all `m.space.child` state events for this room + events, err := w.childReferences(rv.roomID) + if err != nil { + util.GetLogger(w.ctx).WithError(err).WithField("room_id", rv.roomID).Error("failed to extract references for room") + continue + } + discoveredChildEvents = events + + pubRoom := w.publicRoomsChunk(rv.roomID) + + discoveredRooms = append(discoveredRooms, fclient.MSC2946Room{ + PublicRoom: *pubRoom, + RoomType: roomType, + ChildrenState: events, + }) + // don't walk children if the user is not joined/invited to the space + if !isJoinedOrInvited { + continue + } + } else { + // room exists but user is not authorised + continue + } + + // don't walk the children + // if the parent is not a space room + if roomType != ConstCreateEventContentValueSpace { + continue + } + + // For each referenced room ID in the child events being returned to the caller + // add the room ID to the queue of unvisited rooms. Loop from the beginning. + // We need to invert the order here because the child events are lo->hi on the timestamp, + // so we need to ensure we pop in the same lo->hi order, which won't be the case if we + // insert the highest timestamp last in a stack. + for i := len(discoveredChildEvents) - 1; i >= 0; i-- { + spaceContent := struct { + Via []string `json:"via"` + }{} + ev := discoveredChildEvents[i] + _ = json.Unmarshal(ev.Content, &spaceContent) + w.unvisited = append(w.unvisited, roomVisit{ + roomID: ev.StateKey, + parentRoomID: rv.roomID, + depth: rv.depth + 1, + vias: spaceContent.Via, + }) + } + } + + if len(w.unvisited) == 0 { + w.done = true + } + + return discoveredRooms, nil +} + +func (w *RoomHierarchyWalker) Done() bool { + return w.done +} + +func (w *RoomHierarchyWalker) GetCached() roomserver.CachedRoomHierarchyWalker { + return CachedRoomHierarchyWalker{ + rootRoomID: w.rootRoomID, + caller: w.caller, + thisServer: w.thisServer, + rsAPI: w.rsAPI, + fsAPI: w.fsAPI, + ctx: w.ctx, + cache: w.roomHierarchyCache, + suggestedOnly: w.suggestedOnly, + maxDepth: w.maxDepth, + processed: w.processed, + unvisited: w.unvisited, + done: w.done, + } +} + +func (w *RoomHierarchyWalker) stateEvent(roomID, evType, stateKey string) *types.HeaderedEvent { + var queryRes roomserver.QueryCurrentStateResponse + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: evType, + StateKey: stateKey, + } + err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + }, &queryRes) + if err != nil { + return nil + } + return queryRes.StateEvents[tuple] +} + +func (w *RoomHierarchyWalker) roomExists(roomID string) bool { + var queryRes roomserver.QueryServerJoinedToRoomResponse + err := w.rsAPI.QueryServerJoinedToRoom(w.ctx, &roomserver.QueryServerJoinedToRoomRequest{ + RoomID: roomID, + ServerName: w.thisServer, + }, &queryRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") + return false + } + // if the room exists but we aren't in the room then we might have stale data so we want to fetch + // it fresh via federation + return queryRes.RoomExists && queryRes.IsInRoom +} + +// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was +// unsuccessful. +func (w *RoomHierarchyWalker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC2946SpacesResponse { + // only do federated requests for client requests + if w.caller.Device() == nil { + return nil + } + resp, ok := w.roomHierarchyCache.GetRoomHierarchy(roomID) + if ok { + util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID) + return &resp + } + util.GetLogger(w.ctx).Debugf("Querying %s via %+v", roomID, vias) + ctx := context.Background() + // query more of the spaces graph using these servers + for _, serverName := range vias { + if serverName == string(w.thisServer) { + continue + } + res, err := w.fsAPI.MSC2946Spaces(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) + continue + } + // ensure nil slices are empty as we send this to the client sometimes + if res.Room.ChildrenState == nil { + res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} + } + for i := 0; i < len(res.Children); i++ { + child := res.Children[i] + if child.ChildrenState == nil { + child.ChildrenState = []fclient.MSC2946StrippedEvent{} + } + res.Children[i] = child + } + w.roomHierarchyCache.StoreRoomHierarchy(roomID, res) + + return &res + } + return nil +} + +// references returns all child references pointing to or from this room. +func (w *RoomHierarchyWalker) childReferences(roomID string) ([]fclient.MSC2946StrippedEvent, error) { + createTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomCreate, + StateKey: "", + } + var res roomserver.QueryCurrentStateResponse + err := w.rsAPI.QueryCurrentState(context.Background(), &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + AllowWildcards: true, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + createTuple, { + EventType: ConstSpaceChildEventType, + StateKey: "*", + }, + }, + }, &res) + if err != nil { + return nil, err + } + + // don't return any child refs if the room is not a space room + if res.StateEvents[createTuple] != nil { + // escape the `.`s so gjson doesn't think it's nested + roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str + if roomType != ConstCreateEventContentValueSpace { + return []fclient.MSC2946StrippedEvent{}, nil + } + } + delete(res.StateEvents, createTuple) + + el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) + for _, ev := range res.StateEvents { + content := gjson.ParseBytes(ev.Content()) + // only return events that have a `via` key as per MSC1772 + // else we'll incorrectly walk redacted events (as the link + // is in the state_key) + if content.Get("via").Exists() { + strip := stripped(ev.PDU) + if strip == nil { + continue + } + // if suggested only and this child isn't suggested, skip it. + // if suggested only = false we include everything so don't need to check the content. + if w.suggestedOnly && !content.Get("suggested").Bool() { + continue + } + el = append(el, *strip) + } + } + // sort by origin_server_ts as per MSC2946 + sort.Slice(el, func(i, j int) bool { + return el[i].OriginServerTS < el[j].OriginServerTS + }) + + return el, nil +} + +// authorised returns true iff the user is joined this room or the room is world_readable +func (w *RoomHierarchyWalker) authorised(roomID, parentRoomID string) (authed, isJoinedOrInvited bool) { + if clientCaller := w.caller.Device(); clientCaller != nil { + return w.authorisedUser(roomID, clientCaller, parentRoomID) + } else { + return w.authorisedServer(roomID, *w.caller.ServerName()), false + } +} + +// authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable +func (w *RoomHierarchyWalker) authorisedServer(roomID string, callerServerName spec.ServerName) bool { + // Check history visibility / join rules first + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomHistoryVisibility, + StateKey: "", + } + joinRuleTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomJoinRules, + StateKey: "", + } + var queryRoomRes roomserver.QueryCurrentStateResponse + err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + hisVisTuple, joinRuleTuple, + }, + }, &queryRoomRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") + return false + } + hisVisEv := queryRoomRes.StateEvents[hisVisTuple] + if hisVisEv != nil { + hisVis, _ := hisVisEv.HistoryVisibility() + if hisVis == "world_readable" { + return true + } + } + + // check if this room is a restricted room and if so, we need to check if the server is joined to an allowed room ID + // in addition to the actual room ID (but always do the actual one first as it's quicker in the common case) + allowJoinedToRoomIDs := []string{roomID} + joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple] + + if joinRuleEv != nil { + rule, ruleErr := joinRuleEv.JoinRule() + if ruleErr != nil { + util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") + return false + } + + if rule == spec.Public || rule == spec.Knock { + return true + } + + if rule == spec.Restricted { + allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...) + } + } + + // check if server is joined to any allowed room + for _, allowedRoomID := range allowJoinedToRoomIDs { + var queryRes fs.QueryJoinedHostServerNamesInRoomResponse + err = w.fsAPI.QueryJoinedHostServerNamesInRoom(w.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: allowedRoomID, + }, &queryRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") + continue + } + for _, srv := range queryRes.ServerNames { + if srv == callerServerName { + return true + } + } + } + + return false +} + +// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable +// or if the room has a public or knock join rule. +// Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true. +func (w *RoomHierarchyWalker) authorisedUser(roomID string, clientCaller *userapi.Device, parentRoomID string) (authed bool, isJoinedOrInvited bool) { + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomHistoryVisibility, + StateKey: "", + } + joinRuleTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomJoinRules, + StateKey: "", + } + roomMemberTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomMember, + StateKey: clientCaller.UserID, + } + var queryRes roomserver.QueryCurrentStateResponse + err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + hisVisTuple, joinRuleTuple, roomMemberTuple, + }, + }, &queryRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") + return false, false + } + memberEv := queryRes.StateEvents[roomMemberTuple] + if memberEv != nil { + membership, _ := memberEv.Membership() + if membership == spec.Join || membership == spec.Invite { + return true, true + } + } + hisVisEv := queryRes.StateEvents[hisVisTuple] + if hisVisEv != nil { + hisVis, _ := hisVisEv.HistoryVisibility() + if hisVis == "world_readable" { + return true, false + } + } + joinRuleEv := queryRes.StateEvents[joinRuleTuple] + if parentRoomID != "" && joinRuleEv != nil { + var allowed bool + rule, ruleErr := joinRuleEv.JoinRule() + if ruleErr != nil { + util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule") + } else if rule == spec.Public || rule == spec.Knock { + allowed = true + } else if rule == spec.Restricted { + allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership") + // check parent is in the allowed set + for _, a := range allowedRoomIDs { + if parentRoomID == a { + allowed = true + break + } + } + } + if allowed { + // ensure caller is joined to the parent room + var queryRes2 roomserver.QueryCurrentStateResponse + err = w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: parentRoomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + roomMemberTuple, + }, + }, &queryRes2) + if err != nil { + util.GetLogger(w.ctx).WithError(err).WithField("parent_room_id", parentRoomID).Warn("failed to check user is joined to parent room") + } else { + memberEv = queryRes2.StateEvents[roomMemberTuple] + if memberEv != nil { + membership, _ := memberEv.Membership() + if membership == spec.Join { + return true, false + } + } + } + } + } + return false, false +} + +func (w *RoomHierarchyWalker) publicRoomsChunk(roomID string) *fclient.PublicRoom { + pubRooms, err := roomserver.PopulatePublicRooms(w.ctx, []string{roomID}, w.rsAPI) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to PopulatePublicRooms") + return nil + } + if len(pubRooms) == 0 { + return nil + } + return &pubRooms[0] +} + +type roomVisit struct { + roomID string + parentRoomID string + depth int + vias []string // vias to query this room by +} + +func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { + if ev.StateKey() == nil { + return nil + } + return &fclient.MSC2946StrippedEvent{ + Type: ev.Type(), + StateKey: *ev.StateKey(), + Content: ev.Content(), + Sender: string(ev.SenderID()), + OriginServerTS: ev.OriginServerTS(), + } +} + +func (w *RoomHierarchyWalker) restrictedJoinRuleAllowedRooms(joinRuleEv *types.HeaderedEvent, allowType string) (allows []string) { + rule, _ := joinRuleEv.JoinRule() + if rule != spec.Restricted { + return nil + } + var jrContent gomatrixserverlib.JoinRuleContent + if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil { + util.GetLogger(w.ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err) + return nil + } + for _, allow := range jrContent.Allow { + if allow.Type == allowType { + allows = append(allows, allow.RoomID) + } + } + return +} + +// Stripped down version of RoomHierarchyWalker suitable for caching (For pagination purposes) +// +// TODO remove more stuff +type CachedRoomHierarchyWalker struct { + rootRoomID string + caller types.DeviceOrServerName + thisServer spec.ServerName + rsAPI *Queryer + fsAPI fs.RoomserverFederationAPI + ctx context.Context + cache caching.RoomHierarchyCache + suggestedOnly bool + maxDepth int + + processed stringSet + unvisited []roomVisit + + done bool +} + +func (c CachedRoomHierarchyWalker) GetWalker() roomserver.RoomHierarchyWalker { + return &RoomHierarchyWalker{ + rootRoomID: c.rootRoomID, + caller: c.caller, + thisServer: c.thisServer, + rsAPI: c.rsAPI, + fsAPI: c.fsAPI, + ctx: c.ctx, + roomHierarchyCache: c.cache, + suggestedOnly: c.suggestedOnly, + maxDepth: c.maxDepth, + processed: c.processed, + unvisited: c.unvisited, + done: c.done, + } +} + +func (c CachedRoomHierarchyWalker) ValidateParams(suggestedOnly bool, maxDepth int) bool { + return c.suggestedOnly == suggestedOnly && c.maxDepth == maxDepth +} diff --git a/roomserver/types/types.go b/roomserver/types/types.go index 45a3e25fcc..fbff2cdab1 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -22,7 +22,9 @@ import ( "strings" "sync" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "golang.org/x/crypto/blake2b" ) @@ -336,3 +338,36 @@ func (r *RoomInfo) CopyFrom(r2 *RoomInfo) { } var ErrorInvalidRoomInfo = fmt.Errorf("room info is invalid") + +// Struct to represent a device or a server name. +// +// May be used to designate a caller for functions that can be called +// by a client (device) or by a server (server name). +// +// Exactly 1 of Device() and ServerName() will return a non-nil result. +type DeviceOrServerName struct { + device *userapi.Device + serverName *spec.ServerName +} + +func NewDeviceNotServerName(device userapi.Device) DeviceOrServerName { + return DeviceOrServerName{ + device: &device, + serverName: nil, + } +} + +func NewServerNameNotDevice(serverName spec.ServerName) DeviceOrServerName { + return DeviceOrServerName{ + device: nil, + serverName: &serverName, + } +} + +func (s *DeviceOrServerName) Device() *userapi.Device { + return s.device +} + +func (s *DeviceOrServerName) ServerName() *spec.ServerName { + return s.serverName +} diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 3e5ffda925..64b95ffd1e 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -57,7 +57,7 @@ type MSC2946ClientResponse struct { // Enable this MSC func Enable( cfg *config.Dendrite, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, - fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, cache caching.SpaceSummaryRoomsCache, + fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, cache caching.RoomHierarchyCache, ) error { clientAPI := httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(rsAPI, fsAPI, cache, cfg.Global.ServerName), httputil.WithAllowGuests()) routers.Client.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) @@ -87,7 +87,7 @@ func Enable( func federatedSpacesHandler( ctx context.Context, fedReq *fclient.FederationRequest, roomID string, - cache caching.SpaceSummaryRoomsCache, + cache caching.RoomHierarchyCache, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, thisServer spec.ServerName, ) util.JSONResponse { @@ -122,7 +122,7 @@ func federatedSpacesHandler( func spacesHandler( rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, - cache caching.SpaceSummaryRoomsCache, + cache caching.RoomHierarchyCache, thisServer spec.ServerName, ) func(*http.Request, *userapi.Device) util.JSONResponse { // declared outside the returned handler so it persists between calls @@ -168,7 +168,7 @@ type walker struct { rsAPI roomserver.RoomserverInternalAPI fsAPI fs.FederationInternalAPI ctx context.Context - cache caching.SpaceSummaryRoomsCache + cache caching.RoomHierarchyCache suggestedOnly bool limit int maxDepth int @@ -423,7 +423,7 @@ func (w *walker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC294 if w.caller == nil { return nil } - resp, ok := w.cache.GetSpaceSummary(roomID) + resp, ok := w.cache.GetRoomHierarchy(roomID) if ok { util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID) return &resp @@ -451,7 +451,7 @@ func (w *walker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC294 } res.Children[i] = child } - w.cache.StoreSpaceSummary(roomID, res) + w.cache.StoreRoomHierarchy(roomID, res) return &res } From d62a706a44d24b737282070aa71643b5f707ec2a Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Tue, 4 Jul 2023 16:44:55 +0100 Subject: [PATCH 02/21] Add federated route for room/space hierarchy --- federationapi/routing/query.go | 47 ++++++++++++++++++++++++++++++++ federationapi/routing/routing.go | 7 +++++ 2 files changed, 54 insertions(+) diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 2e845f32ca..c8e4664313 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -20,6 +20,7 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -116,3 +117,49 @@ func RoomAliasToID( JSON: resp, } } + +func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationRequest, roomIDStr string, rsAPI roomserverAPI.FederationRoomserverAPI) util.JSONResponse { + parsedRoomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("room is unknown/forbidden"), + } + } + roomID := *parsedRoomID + + suggestedOnly := false // Defaults to false (spec-defined) + switch httpReq.URL.Query().Get("suggested_only") { + case "true": + suggestedOnly = true + case "false": + case "": // Empty string is returned when query param is not set + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"), + } + } + + walker := rsAPI.QueryRoomHierarchy(httpReq.Context(), types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1) + + discoveredRooms, err := walker.NextPage(-1) + + if err != nil { + // TODO + } + + if len(discoveredRooms) == 0 { + return util.JSONResponse{ + Code: 404, + JSON: spec.NotFound("room is unknown/forbidden"), + } + } + return util.JSONResponse{ + Code: 200, + JSON: fclient.MSC2946SpacesResponse{ + Room: discoveredRooms[0], + Children: discoveredRooms[1:], + }, + } +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 4f998821a6..dc7a363e72 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -596,6 +596,13 @@ func Setup( return GetOpenIDUserInfo(req, userAPI) }), ).Methods(http.MethodGet) + + v1fedmux.Handle("/hierarchy/{roomID}", MakeFedAPI( + "federation_room_hierarchy", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, + func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse { + return QueryRoomHierarchy(httpReq, request, vars["roomID"], rsAPI) + }, + )).Methods(http.MethodGet) } func ErrorIfLocalServerNotInRoom( From b9e5e5494d504ec19d6ddf22b9e58db83cf0c3d6 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 6 Jul 2023 12:06:36 +0100 Subject: [PATCH 03/21] Rename some occurences of MSC2946 - still need to rename some identifiers from GMSL --- clientapi/routing/room_hierarchy.go | 4 ++-- federationapi/api/api.go | 2 +- federationapi/internal/federationclient.go | 2 +- roomserver/internal/query/query_room_hierarchy.go | 2 +- setup/mscs/msc2946/msc2946.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index 22b29f487a..639f7b35db 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -136,7 +136,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str return util.JSONResponse{ Code: http.StatusOK, - JSON: MSC2946ClientResponse{ + JSON: RoomHierarchyClientResponse{ Rooms: discoveredRooms, NextBatch: nextBatch, }, @@ -144,7 +144,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str } -type MSC2946ClientResponse struct { +type RoomHierarchyClientResponse struct { Rooms []fclient.MSC2946Room `json:"rooms"` NextBatch string `json:"next_batch,omitempty"` } diff --git a/federationapi/api/api.go b/federationapi/api/api.go index 6d8ae0a120..38cd393758 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -75,7 +75,7 @@ type RoomserverFederationAPI interface { GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error) - MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) + RoomHierarchies(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) } type P2PFederationAPI interface { diff --git a/federationapi/internal/federationclient.go b/federationapi/internal/federationclient.go index d4d7269dbb..d5a45248d9 100644 --- a/federationapi/internal/federationclient.go +++ b/federationapi/internal/federationclient.go @@ -194,7 +194,7 @@ func (a *FederationInternalAPI) MSC2836EventRelationships( return ires.(fclient.MSC2836EventRelationshipsResponse), nil } -func (a *FederationInternalAPI) MSC2946Spaces( +func (a *FederationInternalAPI) RoomHierarchies( ctx context.Context, origin, s spec.ServerName, roomID string, suggestedOnly bool, ) (res fclient.MSC2946SpacesResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index e4ac75e03d..a1d9ae2c42 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -260,7 +260,7 @@ func (w *RoomHierarchyWalker) federatedRoomInfo(roomID string, vias []string) *f if serverName == string(w.thisServer) { continue } - res, err := w.fsAPI.MSC2946Spaces(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) + res, err := w.fsAPI.RoomHierarchies(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) if err != nil { util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) continue diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 64b95ffd1e..3a0c181dc1 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -435,7 +435,7 @@ func (w *walker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC294 if serverName == string(w.thisServer) { continue } - res, err := w.fsAPI.MSC2946Spaces(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) + res, err := w.fsAPI.RoomHierarchies(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) if err != nil { util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) continue From 8642e69568996ea75f0b1dec24092e2c4354904f Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 6 Jul 2023 12:30:27 +0100 Subject: [PATCH 04/21] Rearrange dependencies to avoid nil queryer - A few commits ago we moved `Queryer` to be init when fed API is set - This causes Queryer to be nil before it is set - Causes nil error in `userapi.NewInternalAPI` - Rearrange to avoid this --- cmd/dendrite/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index 7b2bebc0bd..f3140b4e22 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -157,13 +157,14 @@ func main() { keyRing := fsAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) - asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) + + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + rsAPI.SetAppserviceAPI(asAPI) rsAPI.SetUserAPI(userAPI) From 84160593916c69a913e09e84cf6d1c0a5f388db9 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 6 Jul 2023 12:36:07 +0100 Subject: [PATCH 05/21] Fix nil map bugs (by initialising maps) --- clientapi/routing/room_hierarchy.go | 4 +++- roomserver/internal/query/query_room_hierarchy.go | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index 639f7b35db..c15559fd0e 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -34,7 +34,9 @@ type RoomHierarchyPaginationCache struct { } func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache { - return RoomHierarchyPaginationCache{} + return RoomHierarchyPaginationCache{ + cache: map[string]roomserverAPI.CachedRoomHierarchyWalker{}, + } } func (c *RoomHierarchyPaginationCache) Get(token string) roomserverAPI.CachedRoomHierarchyWalker { diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index a1d9ae2c42..1c12d12223 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -37,6 +37,7 @@ func (r *Queryer) QueryRoomHierarchy(ctx context.Context, caller types.DeviceOrS parentRoomID: "", depth: 0, }}, + processed: stringSet{}, } return &walker From 8571b6a01b0955d61c381542c722da9e6e0126e4 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 6 Jul 2023 12:43:36 +0100 Subject: [PATCH 06/21] Update copyright info --- clientapi/routing/room_hierarchy.go | 2 +- roomserver/internal/query/query_room_hierarchy.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index c15559fd0e..e646461e54 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. +// Copyright 2023 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 1c12d12223..9d49622a63 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -1,3 +1,17 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package query import ( From b98e50f5443c28beca3ea1edbdaef1ee2fa549b6 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 6 Jul 2023 13:38:10 +0100 Subject: [PATCH 07/21] Improve room hiearchy error handling + comments --- clientapi/routing/room_hierarchy.go | 18 +++++++++++++++++- clientapi/routing/routing.go | 1 + federationapi/routing/query.go | 18 +++++++++++++++++- roomserver/api/api.go | 11 +++++++++++ roomserver/api/query.go | 8 ++++++++ .../internal/query/query_room_hierarchy.go | 8 ++++++-- 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index e646461e54..c9f53b17e4 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) type RoomHierarchyPaginationCache struct { @@ -54,6 +55,9 @@ func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.CachedRoomHier return token } +// Query the hierarchy of a room/space +// +// Implements /_matrix/client/v1/rooms/{roomID}/hierarchy func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse { parsedRoomID, err := spec.NewRoomID(roomIDStr) if err != nil { @@ -127,7 +131,19 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str discoveredRooms, err := walker.NextPage(limit) if err != nil { - // TODO + switch err.(type) { + case roomserverAPI.ErrRoomUnknownOrNotAllowed: + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("room is unknown/forbidden"), + } + default: + log.WithError(err).Errorf("failed to fetch next page of room hierarchy (CS API)") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } } nextBatch := "" diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index dbd607eaf7..5e691e8ec5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -508,6 +508,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) // Defined outside of handler to persist between calls + // TODO: clear based on some criteria roomHierarchyPaginationCache := new(RoomHierarchyPaginationCache) v1mux.Handle("/rooms/{roomID}/hierarchy", httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index c8e4664313..d52695d0f8 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) // RoomAliasToID converts the queried alias into a room ID and returns it @@ -118,6 +119,9 @@ func RoomAliasToID( } } +// Query the immediate children of a room/space +// +// Implements /_matrix/federation/v1/hierarchy/{roomID} func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationRequest, roomIDStr string, rsAPI roomserverAPI.FederationRoomserverAPI) util.JSONResponse { parsedRoomID, err := spec.NewRoomID(roomIDStr) if err != nil { @@ -146,7 +150,19 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques discoveredRooms, err := walker.NextPage(-1) if err != nil { - // TODO + switch err.(type) { + case roomserverAPI.ErrRoomUnknownOrNotAllowed: + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("room is unknown/forbidden"), + } + default: + log.WithError(err).Errorf("failed to fetch next page of room hierarchy (SS API)") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } } if len(discoveredRooms) == 0 { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index c10a8bdeda..a6e0d5d750 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -34,6 +34,17 @@ func (e ErrNotAllowed) Error() string { return e.Err.Error() } +// ErrRoomUnknownOrNotAllowed is an error return if either the provided +// room ID does not exist, or points to a room that the requester does +// not have access to. +type ErrRoomUnknownOrNotAllowed struct { + Err error +} + +func (e ErrRoomUnknownOrNotAllowed) Error() string { + return e.Err.Error() +} + type RestrictedJoinAPI interface { CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) InvitePending(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (bool, error) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index ded27b8533..a88a402390 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -512,9 +512,17 @@ type QueryRoomHierarchyRequest struct { From int `json:"json"` } +// An iterator-like interface for walking a room/space hierarchy, returning each rooms information. +// +// Used for implementing space summaries / room hierarchies type RoomHierarchyWalker interface { + // Walk the room hierarchy to retrieve room information until either + // no room left, or provided limit reached. If limit provided is -1, then this is + // treated as no limit. NextPage(limit int) ([]fclient.MSC2946Room, error) + // Returns true if there are no more rooms left to walk Done() bool + // Returns a stripped down version of the hiearchy walker suitable for pagination caching GetCached() CachedRoomHierarchyWalker } diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 9d49622a63..1c2eabdcd4 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -17,6 +17,7 @@ package query import ( "context" "encoding/json" + "fmt" "sort" "strings" @@ -92,9 +93,12 @@ const ( ConstSpaceParentEventType = "m.space.parent" ) +// Walk the room hierarchy to retrieve room information until either +// no room left, or provided limit reached. If limit provided is -1, then this is +// treated as no limit. func (w *RoomHierarchyWalker) NextPage(limit int) ([]fclient.MSC2946Room, error) { if authorised, _ := w.authorised(w.rootRoomID, ""); !authorised { - return nil, spec.Forbidden("room is unknown/forbidden") + return nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} } var discoveredRooms []fclient.MSC2946Room @@ -277,7 +281,7 @@ func (w *RoomHierarchyWalker) federatedRoomInfo(roomID string, vias []string) *f } res, err := w.fsAPI.RoomHierarchies(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) if err != nil { - util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) + util.GetLogger(w.ctx).WithError(err).Warnf("failed to call RoomHierarchies on server %s", serverName) continue } // ensure nil slices are empty as we send this to the client sometimes From 97f2befd1bf6e2cf8109d6c37124721b9cac6688 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Wed, 12 Jul 2023 22:14:43 +0100 Subject: [PATCH 08/21] Refactor room hierarchy walker --- clientapi/routing/room_hierarchy.go | 30 +- federationapi/routing/query.go | 5 +- roomserver/api/api.go | 2 +- roomserver/api/query.go | 76 ++- .../internal/query/query_room_hierarchy.go | 514 +++++++----------- 5 files changed, 287 insertions(+), 340 deletions(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index c9f53b17e4..cd53aac330 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -30,24 +30,28 @@ import ( ) type RoomHierarchyPaginationCache struct { - cache map[string]roomserverAPI.CachedRoomHierarchyWalker + cache map[string]roomserverAPI.RoomHierarchyWalker mu sync.Mutex } func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache { return RoomHierarchyPaginationCache{ - cache: map[string]roomserverAPI.CachedRoomHierarchyWalker{}, + cache: map[string]roomserverAPI.RoomHierarchyWalker{}, } } -func (c *RoomHierarchyPaginationCache) Get(token string) roomserverAPI.CachedRoomHierarchyWalker { +func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker { c.mu.Lock() defer c.mu.Unlock() - line := c.cache[token] - return line + line, ok := c.cache[token] + if ok { + return &line + } else { + return nil + } } -func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.CachedRoomHierarchyWalker) string { +func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string { c.mu.Lock() defer c.mu.Unlock() token := uuid.NewString() @@ -114,21 +118,21 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str var walker roomserverAPI.RoomHierarchyWalker if from == "" { // No pagination token provided, so start new hierarchy walker - walker = rsAPI.QueryRoomHierarchy(req.Context(), types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth) + walker = roomserverAPI.NewRoomHierarchyWalker(types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth) } else { // Attempt to resume cached walker cachedWalker := paginationCache.Get(from) - if cachedWalker == nil || !cachedWalker.ValidateParams(suggestedOnly, maxDepth) { + if cachedWalker == nil || cachedWalker.SuggestedOnly != suggestedOnly || cachedWalker.MaxDepth != maxDepth { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"), } } - walker = cachedWalker.GetWalker() + walker = *cachedWalker } - discoveredRooms, err := walker.NextPage(limit) + discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit) if err != nil { switch err.(type) { @@ -147,9 +151,9 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str } nextBatch := "" - if !walker.Done() { - cacheLine := walker.GetCached() - nextBatch = paginationCache.AddLine(cacheLine) + // nextWalker will be nil if there's no more rooms left to walk + if nextWalker != nil { + nextBatch = paginationCache.AddLine(*nextWalker) } return util.JSONResponse{ diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index d52695d0f8..8b37fbe7bf 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -145,9 +145,8 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques } } - walker := rsAPI.QueryRoomHierarchy(httpReq.Context(), types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1) - - discoveredRooms, err := walker.NextPage(-1) + walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1) + discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1) if err != nil { switch err.(type) { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index a6e0d5d750..2162c0666f 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -125,7 +125,7 @@ type QueryEventsAPI interface { } type QueryRoomHierarchyAPI interface { - QueryRoomHierarchy(ctx context.Context, caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) RoomHierarchyWalker + QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.MSC2946Room, *RoomHierarchyWalker, error) } // API functions required by the syncapi diff --git a/roomserver/api/query.go b/roomserver/api/query.go index a88a402390..ca2b6c9222 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -23,7 +23,6 @@ import ( "strings" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" @@ -512,24 +511,63 @@ type QueryRoomHierarchyRequest struct { From int `json:"json"` } -// An iterator-like interface for walking a room/space hierarchy, returning each rooms information. +// A struct storing the intermediate state of a room hierarchy query for pagination purposes. // // Used for implementing space summaries / room hierarchies -type RoomHierarchyWalker interface { - // Walk the room hierarchy to retrieve room information until either - // no room left, or provided limit reached. If limit provided is -1, then this is - // treated as no limit. - NextPage(limit int) ([]fclient.MSC2946Room, error) - // Returns true if there are no more rooms left to walk - Done() bool - // Returns a stripped down version of the hiearchy walker suitable for pagination caching - GetCached() CachedRoomHierarchyWalker -} - -// Stripped down version of RoomHierarchyWalker suitable for caching (for pagination) -type CachedRoomHierarchyWalker interface { - // Converts this cached walker back into an actual walker, to resume walking from. - GetWalker() RoomHierarchyWalker - // Validates that the given parameters match those stored in the cache - ValidateParams(suggestedOnly bool, maxDepth int) bool +// +// Use NewRoomHierarchyWalker on the roomserver API to construct this. +type RoomHierarchyWalker struct { + RootRoomID spec.RoomID + Caller types.DeviceOrServerName + SuggestedOnly bool + MaxDepth int + Processed RoomSet + Unvisited []RoomHierarchyWalkerQueuedRoom +} + +type RoomHierarchyWalkerQueuedRoom struct { + RoomID spec.RoomID + ParentRoomID *spec.RoomID + Depth int + Vias []string // vias to query this room by +} + +func NewRoomHierarchyWalker(caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) RoomHierarchyWalker { + walker := RoomHierarchyWalker{ + RootRoomID: roomID, + Caller: caller, + SuggestedOnly: suggestedOnly, + MaxDepth: maxDepth, + Unvisited: []RoomHierarchyWalkerQueuedRoom{{ + RoomID: roomID, + ParentRoomID: nil, + Depth: 0, + }}, + Processed: NewRoomSet(), + } + + return walker +} + +type RoomSet map[spec.RoomID]struct{} + +func NewRoomSet() RoomSet { + return RoomSet{} +} + +func (s RoomSet) Contains(val spec.RoomID) bool { + _, ok := s[val] + return ok +} + +func (s RoomSet) Add(val spec.RoomID) { + s[val] = struct{}{} +} + +func (s RoomSet) Copy() RoomSet { + copied := make(RoomSet, len(s)) + for k, _ := range s { + copied.Add(k) + } + return copied } diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 1c2eabdcd4..cab552cdc8 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -22,7 +22,6 @@ import ( "strings" fs "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/caching" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -33,59 +32,6 @@ import ( "github.com/tidwall/gjson" ) -// Query the hierarchy of a room (A.K.A 'space summary') -// -// This function returns a iterator-like struct over the hierarchy of a room. -func (r *Queryer) QueryRoomHierarchy(ctx context.Context, caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) roomserver.RoomHierarchyWalker { - walker := RoomHierarchyWalker{ - rootRoomID: roomID.String(), - caller: caller, - thisServer: r.Cfg.Global.ServerName, - rsAPI: r, - fsAPI: r.FSAPI, - ctx: ctx, - roomHierarchyCache: r.Cache, - suggestedOnly: suggestedOnly, - maxDepth: maxDepth, - unvisited: []roomVisit{{ - roomID: roomID.String(), - parentRoomID: "", - depth: 0, - }}, - processed: stringSet{}, - } - - return &walker -} - -type stringSet map[string]struct{} - -func (s stringSet) contains(val string) bool { - _, ok := s[val] - return ok -} - -func (s stringSet) add(val string) { - s[val] = struct{}{} -} - -type RoomHierarchyWalker struct { - rootRoomID string // TODO change to spec.RoomID - caller types.DeviceOrServerName - thisServer spec.ServerName - rsAPI *Queryer - fsAPI fs.RoomserverFederationAPI - ctx context.Context - roomHierarchyCache caching.RoomHierarchyCache - suggestedOnly bool - maxDepth int - - processed stringSet - unvisited []roomVisit - - done bool -} - const ( ConstCreateEventContentKey = "type" ConstCreateEventContentValueSpace = "m.space" @@ -96,34 +42,39 @@ const ( // Walk the room hierarchy to retrieve room information until either // no room left, or provided limit reached. If limit provided is -1, then this is // treated as no limit. -func (w *RoomHierarchyWalker) NextPage(limit int) ([]fclient.MSC2946Room, error) { - if authorised, _ := w.authorised(w.rootRoomID, ""); !authorised { - return nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} +func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.MSC2946Room, *roomserver.RoomHierarchyWalker, error) { + if authorised, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised { + return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} } var discoveredRooms []fclient.MSC2946Room + // Copy unvisited and processed to avoid modifying walker + unvisited := []roomserver.RoomHierarchyWalkerQueuedRoom{} + copy(unvisited, walker.Unvisited) + processed := walker.Processed.Copy() + // Depth first -> stack data structure - for len(w.unvisited) > 0 { + for len(unvisited) > 0 { if len(discoveredRooms) >= limit { break } // pop the stack - rv := w.unvisited[len(w.unvisited)-1] - w.unvisited = w.unvisited[:len(w.unvisited)-1] + queuedRoom := unvisited[len(unvisited)-1] + unvisited = unvisited[:len(unvisited)-1] // If this room has already been processed, skip. // If this room exceeds the specified depth, skip. - if w.processed.contains(rv.roomID) || rv.roomID == "" || (w.maxDepth > 0 && rv.depth > w.maxDepth) { + if processed.Contains(queuedRoom.RoomID) || (walker.MaxDepth > 0 && queuedRoom.Depth > walker.MaxDepth) { continue } // Mark this room as processed. - w.processed.add(rv.roomID) + processed.Add(queuedRoom.RoomID) // if this room is not a space room, skip. var roomType string - create := w.stateEvent(rv.roomID, spec.MRoomCreate, "") + create := stateEvent(ctx, querier, queuedRoom.RoomID, spec.MRoomCreate, "") if create != nil { // escape the `.`s so gjson doesn't think it's nested roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str @@ -134,11 +85,11 @@ func (w *RoomHierarchyWalker) NextPage(limit int) ([]fclient.MSC2946Room, error) // If we know about this room and the caller is authorised (joined/world_readable) then pull // events locally - roomExists := w.roomExists(rv.roomID) + roomExists := roomExists(ctx, querier, queuedRoom.RoomID) if !roomExists { // attempt to query this room over federation, as either we've never heard of it before // or we've left it and hence are not authorised (but info may be exposed regardless) - fedRes := w.federatedRoomInfo(rv.roomID, rv.vias) + fedRes := federatedRoomInfo(ctx, querier, walker.Caller, walker.SuggestedOnly, queuedRoom.RoomID, queuedRoom.Vias) if fedRes != nil { discoveredChildEvents = fedRes.Room.ChildrenState discoveredRooms = append(discoveredRooms, fedRes.Room) @@ -150,16 +101,16 @@ func (w *RoomHierarchyWalker) NextPage(limit int) ([]fclient.MSC2946Room, error) // as these children may be rooms we do know about. roomType = ConstCreateEventContentValueSpace } - } else if authorised, isJoinedOrInvited := w.authorised(rv.roomID, rv.parentRoomID); authorised { + } else if authorised, isJoinedOrInvited := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised { // Get all `m.space.child` state events for this room - events, err := w.childReferences(rv.roomID) + events, err := childReferences(querier, walker.SuggestedOnly, queuedRoom.RoomID) if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("room_id", rv.roomID).Error("failed to extract references for room") + util.GetLogger(ctx).WithError(err).WithField("room_id", queuedRoom.RoomID).Error("failed to extract references for room") continue } discoveredChildEvents = events - pubRoom := w.publicRoomsChunk(rv.roomID) + pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) discoveredRooms = append(discoveredRooms, fclient.MSC2946Room{ PublicRoom: *pubRoom, @@ -192,185 +143,52 @@ func (w *RoomHierarchyWalker) NextPage(limit int) ([]fclient.MSC2946Room, error) }{} ev := discoveredChildEvents[i] _ = json.Unmarshal(ev.Content, &spaceContent) - w.unvisited = append(w.unvisited, roomVisit{ - roomID: ev.StateKey, - parentRoomID: rv.roomID, - depth: rv.depth + 1, - vias: spaceContent.Via, - }) - } - } - - if len(w.unvisited) == 0 { - w.done = true - } - - return discoveredRooms, nil -} - -func (w *RoomHierarchyWalker) Done() bool { - return w.done -} - -func (w *RoomHierarchyWalker) GetCached() roomserver.CachedRoomHierarchyWalker { - return CachedRoomHierarchyWalker{ - rootRoomID: w.rootRoomID, - caller: w.caller, - thisServer: w.thisServer, - rsAPI: w.rsAPI, - fsAPI: w.fsAPI, - ctx: w.ctx, - cache: w.roomHierarchyCache, - suggestedOnly: w.suggestedOnly, - maxDepth: w.maxDepth, - processed: w.processed, - unvisited: w.unvisited, - done: w.done, - } -} - -func (w *RoomHierarchyWalker) stateEvent(roomID, evType, stateKey string) *types.HeaderedEvent { - var queryRes roomserver.QueryCurrentStateResponse - tuple := gomatrixserverlib.StateKeyTuple{ - EventType: evType, - StateKey: stateKey, - } - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, - }, &queryRes) - if err != nil { - return nil - } - return queryRes.StateEvents[tuple] -} -func (w *RoomHierarchyWalker) roomExists(roomID string) bool { - var queryRes roomserver.QueryServerJoinedToRoomResponse - err := w.rsAPI.QueryServerJoinedToRoom(w.ctx, &roomserver.QueryServerJoinedToRoomRequest{ - RoomID: roomID, - ServerName: w.thisServer, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") - return false - } - // if the room exists but we aren't in the room then we might have stale data so we want to fetch - // it fresh via federation - return queryRes.RoomExists && queryRes.IsInRoom -} + childRoomID, err := spec.NewRoomID(ev.StateKey) -// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was -// unsuccessful. -func (w *RoomHierarchyWalker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC2946SpacesResponse { - // only do federated requests for client requests - if w.caller.Device() == nil { - return nil - } - resp, ok := w.roomHierarchyCache.GetRoomHierarchy(roomID) - if ok { - util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID) - return &resp - } - util.GetLogger(w.ctx).Debugf("Querying %s via %+v", roomID, vias) - ctx := context.Background() - // query more of the spaces graph using these servers - for _, serverName := range vias { - if serverName == string(w.thisServer) { - continue - } - res, err := w.fsAPI.RoomHierarchies(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Warnf("failed to call RoomHierarchies on server %s", serverName) - continue - } - // ensure nil slices are empty as we send this to the client sometimes - if res.Room.ChildrenState == nil { - res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} - } - for i := 0; i < len(res.Children); i++ { - child := res.Children[i] - if child.ChildrenState == nil { - child.ChildrenState = []fclient.MSC2946StrippedEvent{} + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("invalid_room_id", ev.StateKey).WithField("parent_room_id", queuedRoom.RoomID).Warn("Invalid room ID in m.space.child state event") + } else { + unvisited = append(unvisited, roomserver.RoomHierarchyWalkerQueuedRoom{ + RoomID: *childRoomID, + ParentRoomID: &queuedRoom.RoomID, + Depth: queuedRoom.Depth + 1, + Vias: spaceContent.Via, + }) } - res.Children[i] = child } - w.roomHierarchyCache.StoreRoomHierarchy(roomID, res) - - return &res - } - return nil -} - -// references returns all child references pointing to or from this room. -func (w *RoomHierarchyWalker) childReferences(roomID string) ([]fclient.MSC2946StrippedEvent, error) { - createTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomCreate, - StateKey: "", - } - var res roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(context.Background(), &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - AllowWildcards: true, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - createTuple, { - EventType: ConstSpaceChildEventType, - StateKey: "*", - }, - }, - }, &res) - if err != nil { - return nil, err } - // don't return any child refs if the room is not a space room - if res.StateEvents[createTuple] != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str - if roomType != ConstCreateEventContentValueSpace { - return []fclient.MSC2946StrippedEvent{}, nil + if len(unvisited) == 0 { + // If no more rooms to walk, then don't return a walker for future pages + return discoveredRooms, nil, nil + } else { + // If there are more rooms to walk, then return a new walker to resume walking from (for querying more pages) + newWalker := roomserver.RoomHierarchyWalker{ + RootRoomID: walker.RootRoomID, + Caller: walker.Caller, + SuggestedOnly: walker.SuggestedOnly, + MaxDepth: walker.MaxDepth, + Unvisited: unvisited, + Processed: processed, } - } - delete(res.StateEvents, createTuple) - el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) - for _, ev := range res.StateEvents { - content := gjson.ParseBytes(ev.Content()) - // only return events that have a `via` key as per MSC1772 - // else we'll incorrectly walk redacted events (as the link - // is in the state_key) - if content.Get("via").Exists() { - strip := stripped(ev.PDU) - if strip == nil { - continue - } - // if suggested only and this child isn't suggested, skip it. - // if suggested only = false we include everything so don't need to check the content. - if w.suggestedOnly && !content.Get("suggested").Bool() { - continue - } - el = append(el, *strip) - } + return discoveredRooms, &newWalker, nil } - // sort by origin_server_ts as per MSC2946 - sort.Slice(el, func(i, j int) bool { - return el[i].OriginServerTS < el[j].OriginServerTS - }) - return el, nil } // authorised returns true iff the user is joined this room or the room is world_readable -func (w *RoomHierarchyWalker) authorised(roomID, parentRoomID string) (authed, isJoinedOrInvited bool) { - if clientCaller := w.caller.Device(); clientCaller != nil { - return w.authorisedUser(roomID, clientCaller, parentRoomID) +func authorised(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed, isJoinedOrInvited bool) { + if clientCaller := caller.Device(); clientCaller != nil { + return authorisedUser(ctx, querier, clientCaller, roomID, parentRoomID) } else { - return w.authorisedServer(roomID, *w.caller.ServerName()), false + return authorisedServer(ctx, querier, roomID, *caller.ServerName()), false } } // authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable -func (w *RoomHierarchyWalker) authorisedServer(roomID string, callerServerName spec.ServerName) bool { +func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, callerServerName spec.ServerName) bool { // Check history visibility / join rules first hisVisTuple := gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomHistoryVisibility, @@ -381,14 +199,14 @@ func (w *RoomHierarchyWalker) authorisedServer(roomID string, callerServerName s StateKey: "", } var queryRoomRes roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, + err := querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), StateTuples: []gomatrixserverlib.StateKeyTuple{ hisVisTuple, joinRuleTuple, }, }, &queryRoomRes) if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") + util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState") return false } hisVisEv := queryRoomRes.StateEvents[hisVisTuple] @@ -401,13 +219,13 @@ func (w *RoomHierarchyWalker) authorisedServer(roomID string, callerServerName s // check if this room is a restricted room and if so, we need to check if the server is joined to an allowed room ID // in addition to the actual room ID (but always do the actual one first as it's quicker in the common case) - allowJoinedToRoomIDs := []string{roomID} + allowJoinedToRoomIDs := []spec.RoomID{roomID} joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple] if joinRuleEv != nil { rule, ruleErr := joinRuleEv.JoinRule() if ruleErr != nil { - util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") + util.GetLogger(ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") return false } @@ -416,18 +234,18 @@ func (w *RoomHierarchyWalker) authorisedServer(roomID string, callerServerName s } if rule == spec.Restricted { - allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...) + allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, restrictedJoinRuleAllowedRooms(ctx, joinRuleEv, "m.room_membership")...) } } // check if server is joined to any allowed room for _, allowedRoomID := range allowJoinedToRoomIDs { var queryRes fs.QueryJoinedHostServerNamesInRoomResponse - err = w.fsAPI.QueryJoinedHostServerNamesInRoom(w.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: allowedRoomID, + err = querier.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: allowedRoomID.String(), }, &queryRes) if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") + util.GetLogger(ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") continue } for _, srv := range queryRes.ServerNames { @@ -443,7 +261,7 @@ func (w *RoomHierarchyWalker) authorisedServer(roomID string, callerServerName s // authorisedUser returns true iff the user is invited/joined this room or the room is world_readable // or if the room has a public or knock join rule. // Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true. -func (w *RoomHierarchyWalker) authorisedUser(roomID string, clientCaller *userapi.Device, parentRoomID string) (authed bool, isJoinedOrInvited bool) { +func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi.Device, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed bool, isJoinedOrInvited bool) { hisVisTuple := gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomHistoryVisibility, StateKey: "", @@ -457,14 +275,14 @@ func (w *RoomHierarchyWalker) authorisedUser(roomID string, clientCaller *userap StateKey: clientCaller.UserID, } var queryRes roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, + err := querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), StateTuples: []gomatrixserverlib.StateKeyTuple{ hisVisTuple, joinRuleTuple, roomMemberTuple, }, }, &queryRes) if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") + util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState") return false, false } memberEv := queryRes.StateEvents[roomMemberTuple] @@ -482,18 +300,18 @@ func (w *RoomHierarchyWalker) authorisedUser(roomID string, clientCaller *userap } } joinRuleEv := queryRes.StateEvents[joinRuleTuple] - if parentRoomID != "" && joinRuleEv != nil { + if parentRoomID != nil && joinRuleEv != nil { var allowed bool rule, ruleErr := joinRuleEv.JoinRule() if ruleErr != nil { - util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule") + util.GetLogger(ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule") } else if rule == spec.Public || rule == spec.Knock { allowed = true } else if rule == spec.Restricted { - allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership") + allowedRoomIDs := restrictedJoinRuleAllowedRooms(ctx, joinRuleEv, "m.room_membership") // check parent is in the allowed set for _, a := range allowedRoomIDs { - if parentRoomID == a { + if *parentRoomID == a { allowed = true break } @@ -502,14 +320,14 @@ func (w *RoomHierarchyWalker) authorisedUser(roomID string, clientCaller *userap if allowed { // ensure caller is joined to the parent room var queryRes2 roomserver.QueryCurrentStateResponse - err = w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: parentRoomID, + err = querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: parentRoomID.String(), StateTuples: []gomatrixserverlib.StateKeyTuple{ roomMemberTuple, }, }, &queryRes2) if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("parent_room_id", parentRoomID).Warn("failed to check user is joined to parent room") + util.GetLogger(ctx).WithError(err).WithField("parent_room_id", parentRoomID).Warn("failed to check user is joined to parent room") } else { memberEv = queryRes2.StateEvents[roomMemberTuple] if memberEv != nil { @@ -524,23 +342,147 @@ func (w *RoomHierarchyWalker) authorisedUser(roomID string, clientCaller *userap return false, false } -func (w *RoomHierarchyWalker) publicRoomsChunk(roomID string) *fclient.PublicRoom { - pubRooms, err := roomserver.PopulatePublicRooms(w.ctx, []string{roomID}, w.rsAPI) +func stateEvent(ctx context.Context, querier *Queryer, roomID spec.RoomID, evType, stateKey string) *types.HeaderedEvent { + var queryRes roomserver.QueryCurrentStateResponse + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: evType, + StateKey: stateKey, + } + err := querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), + StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + }, &queryRes) if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to PopulatePublicRooms") return nil } - if len(pubRooms) == 0 { + return queryRes.StateEvents[tuple] +} + +func roomExists(ctx context.Context, querier *Queryer, roomID spec.RoomID) bool { + var queryRes roomserver.QueryServerJoinedToRoomResponse + err := querier.QueryServerJoinedToRoom(ctx, &roomserver.QueryServerJoinedToRoomRequest{ + RoomID: roomID.String(), + ServerName: querier.Cfg.Global.ServerName, + }, &queryRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") + return false + } + // if the room exists but we aren't in the room then we might have stale data so we want to fetch + // it fresh via federation + return queryRes.RoomExists && queryRes.IsInRoom +} + +// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was +// unsuccessful. +func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, suggestedOnly bool, roomID spec.RoomID, vias []string) *fclient.MSC2946SpacesResponse { + // only do federated requests for client requests + if caller.Device() == nil { return nil } - return &pubRooms[0] + resp, ok := querier.Cache.GetRoomHierarchy(roomID.String()) + if ok { + util.GetLogger(ctx).Debugf("Returning cached response for %s", roomID) + return &resp + } + util.GetLogger(ctx).Debugf("Querying %s via %+v", roomID, vias) + innerCtx := context.Background() + // query more of the spaces graph using these servers + for _, serverName := range vias { + if serverName == string(querier.Cfg.Global.ServerName) { + continue + } + res, err := querier.FSAPI.RoomHierarchies(innerCtx, querier.Cfg.Global.ServerName, spec.ServerName(serverName), roomID.String(), suggestedOnly) + if err != nil { + util.GetLogger(ctx).WithError(err).Warnf("failed to call RoomHierarchies on server %s", serverName) + continue + } + // ensure nil slices are empty as we send this to the client sometimes + if res.Room.ChildrenState == nil { + res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} + } + for i := 0; i < len(res.Children); i++ { + child := res.Children[i] + if child.ChildrenState == nil { + child.ChildrenState = []fclient.MSC2946StrippedEvent{} + } + res.Children[i] = child + } + querier.Cache.StoreRoomHierarchy(roomID.String(), res) + + return &res + } + return nil } -type roomVisit struct { - roomID string - parentRoomID string - depth int - vias []string // vias to query this room by +// references returns all child references pointing to or from this room. +func childReferences(querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ([]fclient.MSC2946StrippedEvent, error) { + createTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomCreate, + StateKey: "", + } + var res roomserver.QueryCurrentStateResponse + err := querier.QueryCurrentState(context.Background(), &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), + AllowWildcards: true, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + createTuple, { + EventType: ConstSpaceChildEventType, + StateKey: "*", + }, + }, + }, &res) + if err != nil { + return nil, err + } + + // don't return any child refs if the room is not a space room + if res.StateEvents[createTuple] != nil { + // escape the `.`s so gjson doesn't think it's nested + roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str + if roomType != ConstCreateEventContentValueSpace { + return []fclient.MSC2946StrippedEvent{}, nil + } + } + delete(res.StateEvents, createTuple) + + el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) + for _, ev := range res.StateEvents { + content := gjson.ParseBytes(ev.Content()) + // only return events that have a `via` key as per MSC1772 + // else we'll incorrectly walk redacted events (as the link + // is in the state_key) + if content.Get("via").Exists() { + strip := stripped(ev.PDU) + if strip == nil { + continue + } + // if suggested only and this child isn't suggested, skip it. + // if suggested only = false we include everything so don't need to check the content. + if suggestedOnly && !content.Get("suggested").Bool() { + continue + } + el = append(el, *strip) + } + } + // sort by origin_server_ts as per MSC2946 + sort.Slice(el, func(i, j int) bool { + return el[i].OriginServerTS < el[j].OriginServerTS + }) + + return el, nil +} + +func publicRoomsChunk(ctx context.Context, querier *Queryer, roomID spec.RoomID) *fclient.PublicRoom { + pubRooms, err := roomserver.PopulatePublicRooms(ctx, []string{roomID.String()}, querier) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to PopulatePublicRooms") + return nil + } + if len(pubRooms) == 0 { + return nil + } + return &pubRooms[0] } func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { @@ -556,61 +498,25 @@ func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { } } -func (w *RoomHierarchyWalker) restrictedJoinRuleAllowedRooms(joinRuleEv *types.HeaderedEvent, allowType string) (allows []string) { +func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.HeaderedEvent, allowType string) (allows []spec.RoomID) { rule, _ := joinRuleEv.JoinRule() if rule != spec.Restricted { return nil } var jrContent gomatrixserverlib.JoinRuleContent if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil { - util.GetLogger(w.ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err) + util.GetLogger(ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err) return nil } for _, allow := range jrContent.Allow { if allow.Type == allowType { - allows = append(allows, allow.RoomID) + allowedRoomID, err := spec.NewRoomID(allow.RoomID) + if err != nil { + util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID(), err) + } else { + allows = append(allows, *allowedRoomID) + } } } return } - -// Stripped down version of RoomHierarchyWalker suitable for caching (For pagination purposes) -// -// TODO remove more stuff -type CachedRoomHierarchyWalker struct { - rootRoomID string - caller types.DeviceOrServerName - thisServer spec.ServerName - rsAPI *Queryer - fsAPI fs.RoomserverFederationAPI - ctx context.Context - cache caching.RoomHierarchyCache - suggestedOnly bool - maxDepth int - - processed stringSet - unvisited []roomVisit - - done bool -} - -func (c CachedRoomHierarchyWalker) GetWalker() roomserver.RoomHierarchyWalker { - return &RoomHierarchyWalker{ - rootRoomID: c.rootRoomID, - caller: c.caller, - thisServer: c.thisServer, - rsAPI: c.rsAPI, - fsAPI: c.fsAPI, - ctx: c.ctx, - roomHierarchyCache: c.cache, - suggestedOnly: c.suggestedOnly, - maxDepth: c.maxDepth, - processed: c.processed, - unvisited: c.unvisited, - done: c.done, - } -} - -func (c CachedRoomHierarchyWalker) ValidateParams(suggestedOnly bool, maxDepth int) bool { - return c.suggestedOnly == suggestedOnly && c.maxDepth == maxDepth -} From ad1de7d9bbba7beef827a92827c913eb466392fc Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 13 Jul 2023 14:47:10 +0100 Subject: [PATCH 09/21] add comments --- clientapi/routing/room_hierarchy.go | 5 +++++ roomserver/api/api.go | 7 ++++++ roomserver/api/query.go | 10 ++++++++- .../internal/query/query_room_hierarchy.go | 22 +++++++++++++------ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index cd53aac330..a1c2a610ae 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -29,17 +29,20 @@ import ( log "github.com/sirupsen/logrus" ) +// For storing pagination information for room hierarchies type RoomHierarchyPaginationCache struct { cache map[string]roomserverAPI.RoomHierarchyWalker mu sync.Mutex } +// Create a new, empty, pagination cache. func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache { return RoomHierarchyPaginationCache{ cache: map[string]roomserverAPI.RoomHierarchyWalker{}, } } +// Get a cached page, or nil if there is no associated page in the cache. func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker { c.mu.Lock() defer c.mu.Unlock() @@ -51,6 +54,7 @@ func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHier } } +// Add a cache line to the pagination cache. func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string { c.mu.Lock() defer c.mu.Unlock() @@ -166,6 +170,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str } +// Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy type RoomHierarchyClientResponse struct { Rooms []fclient.MSC2946Room `json:"rooms"` NextBatch string `json:"next_batch,omitempty"` diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 2162c0666f..826bb06117 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -125,6 +125,13 @@ type QueryEventsAPI interface { } type QueryRoomHierarchyAPI interface { + // Traverse the room hierarchy using the provided walker up to the provided limit, + // returning a new walker which can be used to fetch the next page. + // + // If limit is -1, this is treated as no limit, and the entire hierarchy will be traversed. + // + // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it + // can be cached. QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.MSC2946Room, *RoomHierarchyWalker, error) } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index ca2b6c9222..c3d6c303bb 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -515,7 +515,8 @@ type QueryRoomHierarchyRequest struct { // // Used for implementing space summaries / room hierarchies // -// Use NewRoomHierarchyWalker on the roomserver API to construct this. +// Use NewRoomHierarchyWalker to construct this, and QueryNextRoomHierarchyPage on the roomserver API +// to traverse the room hierarchy. type RoomHierarchyWalker struct { RootRoomID spec.RoomID Caller types.DeviceOrServerName @@ -532,6 +533,9 @@ type RoomHierarchyWalkerQueuedRoom struct { Vias []string // vias to query this room by } +// Create a new room hierarchy walker, starting from the provided root room ID. +// +// Use the resulting struct with QueryNextRoomHierarchyPage on the roomserver API to traverse the room hierarchy. func NewRoomHierarchyWalker(caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) RoomHierarchyWalker { walker := RoomHierarchyWalker{ RootRoomID: roomID, @@ -549,17 +553,21 @@ func NewRoomHierarchyWalker(caller types.DeviceOrServerName, roomID spec.RoomID, return walker } +// A set of room IDs. type RoomSet map[spec.RoomID]struct{} +// Create a new empty room set. func NewRoomSet() RoomSet { return RoomSet{} } +// Check if a room ID is in a room set. func (s RoomSet) Contains(val spec.RoomID) bool { _, ok := s[val] return ok } +// Add a room ID to a room set. func (s RoomSet) Add(val spec.RoomID) { s[val] = struct{}{} } diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index cab552cdc8..4b1986d4b7 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -39,9 +39,13 @@ const ( ConstSpaceParentEventType = "m.space.parent" ) -// Walk the room hierarchy to retrieve room information until either -// no room left, or provided limit reached. If limit provided is -1, then this is -// treated as no limit. +// Traverse the room hierarchy using the provided walker up to the provided limit, +// returning a new walker which can be used to fetch the next page. +// +// If limit is -1, this is treated as no limit, and the entire hierarchy will be traversed. +// +// If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it +// can be cached. func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.MSC2946Room, *roomserver.RoomHierarchyWalker, error) { if authorised, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised { return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} @@ -234,7 +238,7 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, } if rule == spec.Restricted { - allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, restrictedJoinRuleAllowedRooms(ctx, joinRuleEv, "m.room_membership")...) + allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, restrictedJoinRuleAllowedRooms(ctx, joinRuleEv)...) } } @@ -308,7 +312,7 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi } else if rule == spec.Public || rule == spec.Knock { allowed = true } else if rule == spec.Restricted { - allowedRoomIDs := restrictedJoinRuleAllowedRooms(ctx, joinRuleEv, "m.room_membership") + allowedRoomIDs := restrictedJoinRuleAllowedRooms(ctx, joinRuleEv) // check parent is in the allowed set for _, a := range allowedRoomIDs { if *parentRoomID == a { @@ -342,6 +346,7 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi return false, false } +// helper function to fetch a state event func stateEvent(ctx context.Context, querier *Queryer, roomID spec.RoomID, evType, stateKey string) *types.HeaderedEvent { var queryRes roomserver.QueryCurrentStateResponse tuple := gomatrixserverlib.StateKeyTuple{ @@ -358,6 +363,7 @@ func stateEvent(ctx context.Context, querier *Queryer, roomID spec.RoomID, evTyp return queryRes.StateEvents[tuple] } +// returns true if the current server is participating in the provided room func roomExists(ctx context.Context, querier *Queryer, roomID spec.RoomID) bool { var queryRes roomserver.QueryServerJoinedToRoomResponse err := querier.QueryServerJoinedToRoom(ctx, &roomserver.QueryServerJoinedToRoomRequest{ @@ -473,6 +479,7 @@ func childReferences(querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ( return el, nil } +// fetch public room information for provided room func publicRoomsChunk(ctx context.Context, querier *Queryer, roomID spec.RoomID) *fclient.PublicRoom { pubRooms, err := roomserver.PopulatePublicRooms(ctx, []string{roomID.String()}, querier) if err != nil { @@ -498,7 +505,8 @@ func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { } } -func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.HeaderedEvent, allowType string) (allows []spec.RoomID) { +// given join_rule event, return list of rooms where membership of that room allows joining. +func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.HeaderedEvent) (allows []spec.RoomID) { rule, _ := joinRuleEv.JoinRule() if rule != spec.Restricted { return nil @@ -509,7 +517,7 @@ func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.Heade return nil } for _, allow := range jrContent.Allow { - if allow.Type == allowType { + if allow.Type == spec.MRoomMembership { allowedRoomID, err := spec.NewRoomID(allow.RoomID) if err != nil { util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID(), err) From 05e331d6f60239cb7b965a7f62a7307a6c5da7ac Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Fri, 14 Jul 2023 13:12:04 +0100 Subject: [PATCH 10/21] Use values from GMSL --- go.mod | 2 +- go.sum | 6 +++ .../internal/query/query_room_hierarchy.go | 38 +++++++++---------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 08ebb623e0..7072ec81d3 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230707183936-226d2080393a + github.com/matrix-org/gomatrixserverlib v0.0.0-20230713175830-cf82e01c0017 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index 3c1c327cfb..169da68860 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -175,12 +176,14 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= @@ -209,6 +212,8 @@ github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20230707183936-226d2080393a h1:jDoCCEUPnAyPOXO76V4lS1H92gfOO1orMy805gf25bg= github.com/matrix-org/gomatrixserverlib v0.0.0-20230707183936-226d2080393a/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230713175830-cf82e01c0017 h1:5ArSOagq7kDqw+mzVcCB55LDZRgWfH0R5b874GoKYvo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230713175830-cf82e01c0017/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= @@ -241,6 +246,7 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA= diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 4b1986d4b7..d10cae25e4 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "sort" - "strings" fs "github.com/matrix-org/dendrite/federationapi/api" roomserver "github.com/matrix-org/dendrite/roomserver/api" @@ -32,13 +31,6 @@ import ( "github.com/tidwall/gjson" ) -const ( - ConstCreateEventContentKey = "type" - ConstCreateEventContentValueSpace = "m.space" - ConstSpaceChildEventType = "m.space.child" - ConstSpaceParentEventType = "m.space.parent" -) - // Traverse the room hierarchy using the provided walker up to the provided limit, // returning a new walker which can be used to fetch the next page. // @@ -80,8 +72,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r var roomType string create := stateEvent(ctx, querier, queuedRoom.RoomID, spec.MRoomCreate, "") if create != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str + var createContent gomatrixserverlib.CreateContent + err := json.Unmarshal(create.Content(), &createContent) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("create_content", create.Content()).Warn("failed to unmarshal m.room.create event") + } + roomType = createContent.RoomType } // Collect rooms/events to send back (either locally or fetched via federation) @@ -103,11 +99,11 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r // mark this room as a space room as the federated server responded. // we need to do this so we add the children of this room to the unvisited stack // as these children may be rooms we do know about. - roomType = ConstCreateEventContentValueSpace + roomType = spec.MSpace } } else if authorised, isJoinedOrInvited := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised { // Get all `m.space.child` state events for this room - events, err := childReferences(querier, walker.SuggestedOnly, queuedRoom.RoomID) + events, err := childReferences(ctx, querier, walker.SuggestedOnly, queuedRoom.RoomID) if err != nil { util.GetLogger(ctx).WithError(err).WithField("room_id", queuedRoom.RoomID).Error("failed to extract references for room") continue @@ -132,7 +128,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r // don't walk the children // if the parent is not a space room - if roomType != ConstCreateEventContentValueSpace { + if roomType != spec.MSpace { continue } @@ -422,7 +418,7 @@ func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.Devic } // references returns all child references pointing to or from this room. -func childReferences(querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ([]fclient.MSC2946StrippedEvent, error) { +func childReferences(ctx context.Context, querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ([]fclient.MSC2946StrippedEvent, error) { createTuple := gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomCreate, StateKey: "", @@ -433,7 +429,7 @@ func childReferences(querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ( AllowWildcards: true, StateTuples: []gomatrixserverlib.StateKeyTuple{ createTuple, { - EventType: ConstSpaceChildEventType, + EventType: spec.MSpaceChild, StateKey: "*", }, }, @@ -443,10 +439,14 @@ func childReferences(querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ( } // don't return any child refs if the room is not a space room - if res.StateEvents[createTuple] != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str - if roomType != ConstCreateEventContentValueSpace { + if create := res.StateEvents[createTuple]; create != nil { + var createContent gomatrixserverlib.CreateContent + err := json.Unmarshal(create.Content(), &createContent) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("create_content", create.Content()).Warn("failed to unmarshal m.room.create event") + } + roomType := createContent.RoomType + if roomType != spec.MSpace { return []fclient.MSC2946StrippedEvent{}, nil } } From 530f42e5453646f2609350cc3d2e5bb288cc35ef Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Fri, 14 Jul 2023 13:26:08 +0100 Subject: [PATCH 11/21] Fix styling --- clientapi/routing/room_hierarchy.go | 6 ++++-- roomserver/api/query.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index a1c2a610ae..e314d07c82 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -92,7 +92,8 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str limit := 1000 // Default to 1000 limitStr := req.URL.Query().Get("limit") if limitStr != "" { - maybeLimit, err := strconv.Atoi(limitStr) + var maybeLimit int + maybeLimit, err = strconv.Atoi(limitStr) if err != nil || maybeLimit < 0 { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -108,7 +109,8 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str maxDepth := -1 // '-1' representing no maximum depth maxDepthStr := req.URL.Query().Get("max_depth") if maxDepthStr != "" { - maybeMaxDepth, err := strconv.Atoi(maxDepthStr) + var maybeMaxDepth int + maybeMaxDepth, err = strconv.Atoi(maxDepthStr) if err != nil || maybeMaxDepth < 0 { return util.JSONResponse{ Code: http.StatusBadRequest, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index c3d6c303bb..57bac2df95 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -574,7 +574,7 @@ func (s RoomSet) Add(val spec.RoomID) { func (s RoomSet) Copy() RoomSet { copied := make(RoomSet, len(s)) - for k, _ := range s { + for k := range s { copied.Add(k) } return copied From a9ee49e6ee90d214a52c03428988ab7e612e1ccc Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Fri, 14 Jul 2023 14:45:05 +0100 Subject: [PATCH 12/21] Update tests to init roomserver properly --- appservice/appservice_test.go | 3 ++- clientapi/admin_test.go | 18 ++++++++++++++---- clientapi/clientapi_test.go | 9 +++++++++ clientapi/routing/joinroom_test.go | 2 +- clientapi/routing/login_test.go | 1 + clientapi/routing/register_test.go | 3 +++ .../monolith/monolith.go | 3 +-- roomserver/roomserver_test.go | 7 ++++--- 8 files changed, 35 insertions(+), 11 deletions(-) diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 878ca5666e..ddc24477bf 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -134,7 +134,6 @@ func TestAppserviceInternalAPI(t *testing.T) { } as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} - t.Cleanup(func() { ctx.ShutdownDendrite() ctx.WaitForShutdown() @@ -144,6 +143,7 @@ func TestAppserviceInternalAPI(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) @@ -238,6 +238,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 9d2acd68ed..66667b03ca 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -44,6 +44,7 @@ func TestAdminCreateToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -194,6 +195,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -311,6 +313,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -411,6 +414,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -504,6 +508,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -686,6 +691,7 @@ func TestAdminResetPassword(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. @@ -780,13 +786,14 @@ func TestPurgeRoom(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -851,12 +858,13 @@ func TestAdminEvacuateRoom(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -951,12 +959,13 @@ func TestAdminEvacuateUser(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -1045,6 +1054,7 @@ func TestAdminMarkAsStale(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index b339818a4c..ae14d271da 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -120,6 +120,7 @@ func TestGetPutDevices(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -168,6 +169,7 @@ func TestDeleteDevice(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. @@ -272,6 +274,7 @@ func TestDeleteDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. @@ -947,6 +950,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -993,6 +997,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. @@ -1090,6 +1095,7 @@ func Test3PID(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1265,6 +1271,7 @@ func TestPushRules(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -1651,6 +1658,7 @@ func TestKeys(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -2112,6 +2120,7 @@ func TestKeyBackup(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 0ddff8a95b..933ea8d3af 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -35,9 +35,9 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc // Create the users in the userapi for _, u := range []*test.User{alice, bob, charlie} { diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index bff676826d..252017db2e 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -47,6 +47,7 @@ func TestLogin(t *testing.T) { routers := httputil.NewRouters() caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // Needed for /login userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 2a88ec3806..0a1986cf71 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -415,6 +415,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) for _, tc := range testCases { @@ -594,6 +595,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" @@ -634,6 +636,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) expectedDisplayName := "rabbit" diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 397473865f..dfd139e57e 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -143,13 +143,12 @@ func (p *P2PMonolith) SetupDendrite( fsAPI := federationapi.NewInternalAPI( processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) + rsAPI.SetFederationAPI(fsAPI, keyRing) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetFederationAPI(fsAPI, keyRing) - userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation) roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation) diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 76b21ad23f..ce0721bea5 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -249,13 +249,14 @@ func TestPurgeRoom(t *testing.T) { defer jetstream.DeleteAllStreams(jsCtx, &cfg.Global.JetStream) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + // Create the room if err = api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -1035,8 +1036,8 @@ func TestUpgrade(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { From 14f8b302918d24ba04010921b16be42d94dda024 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Fri, 14 Jul 2023 15:04:43 +0100 Subject: [PATCH 13/21] Remove MSC2946 as an MSC option --- .../monolith/monolith.go | 2 +- cmd/dendrite-demo-yggdrasil/main.go | 2 +- cmd/generate-config/main.go | 2 +- dendrite-sample.yaml | 1 - docs/FAQ.md | 6 +- helm/dendrite/README.md | 246 +++--- helm/dendrite/values.yaml | 4 +- setup/config/config_mscs.go | 1 - setup/mscs/msc2946/msc2946.go | 744 ------------------ setup/mscs/mscs.go | 3 - 10 files changed, 129 insertions(+), 882 deletions(-) delete mode 100644 setup/mscs/msc2946/msc2946.go diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index dfd139e57e..4fbb70f2a7 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -98,7 +98,7 @@ func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir st cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(storageDir, dbPrefix))) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(storageDir, dbPrefix))) cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(storageDir, dbPrefix))) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(storageDir, dbPrefix))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 25c1475cbe..3ec5501139 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -134,7 +134,7 @@ func main() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName))) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName))) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName))) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index cb57ed78fb..2379ce2bb9 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -74,7 +74,7 @@ func main() { // don't hit matrix.org when running tests!!! cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{} cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media")) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"} + cfg.MSCs.MSCs = []string{"msc2836", "msc2444", "msc2753"} cfg.Logging[0].Level = "trace" cfg.Logging[0].Type = "std" cfg.UserAPI.BCryptCost = bcrypt.MinCost diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 96143d85f2..8abc230115 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -276,7 +276,6 @@ media_api: mscs: mscs: # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) # Configuration for the Sync API. sync_api: diff --git a/docs/FAQ.md b/docs/FAQ.md index 757bf96255..0273d53823 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -64,16 +64,14 @@ Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially suppo ## Does Dendrite support Space Summaries? -Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration: +Yes, to enable them [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. ``` mscs: mscs: - - msc2946 + - msc2836 ``` -Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. - Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject. ## Does Dendrite support push notifications? diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 562d1e2359..53cb187ab2 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -35,131 +35,131 @@ Create a folder `appservices` and place your configurations in there. The confi * ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | +| Repository | Name | Version | +| ---------------------------------- | ---------- | ------- | +| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | ## Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| image.repository | string | `"ghcr.io/matrix-org/dendrite-monolith"` | Docker repository/image to use | -| image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | -| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | -| signing_key.create | bool | `true` | Create a new signing key, if not exists | -| signing_key.existingSecret | string | `""` | Use an existing secret | -| resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | -| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | -| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | -| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | -| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | -| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | -| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | -| extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | -| extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | -| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | -| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | -| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | -| dendrite_config.version | int | `2` | | -| dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | -| dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | -| dendrite_config.global.well_known_server_name | string | `""` | The server name to delegate server-server communications to, with optional port e.g. localhost:443 | -| dendrite_config.global.well_known_client_name | string | `""` | The server name to delegate client-server communications to, with optional port e.g. localhost:443 | -| dendrite_config.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Lists of domains that the server will trust as identity servers to verify third party identifiers such as phone numbers and email addresses. | -| dendrite_config.global.old_private_keys | string | `nil` | The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) to old signing keys that were formerly in use on this domain name. These keys will not be used for federation request or event signing, but will be provided to any other homeserver that asks when trying to verify old events. | -| dendrite_config.global.disable_federation | bool | `false` | Disable federation. Dendrite will not be able to make any outbound HTTP requests to other servers and the federation API will not be exposed. | -| dendrite_config.global.key_validity_period | string | `"168h0m0s"` | | -| dendrite_config.global.database.connection_string | string | `""` | The connection string for connections to Postgres. This will be set automatically if using the Postgres dependency | -| dendrite_config.global.database.max_open_conns | int | `90` | Default database maximum open connections | -| dendrite_config.global.database.max_idle_conns | int | `5` | Default database maximum idle connections | -| dendrite_config.global.database.conn_max_lifetime | int | `-1` | Default database maximum lifetime | -| dendrite_config.global.jetstream.storage_path | string | `"/data/jetstream"` | Persistent directory to store JetStream streams in. | -| dendrite_config.global.jetstream.addresses | list | `[]` | NATS JetStream server addresses if not using internal NATS. | -| dendrite_config.global.jetstream.topic_prefix | string | `"Dendrite"` | The prefix for JetStream streams | -| dendrite_config.global.jetstream.in_memory | bool | `false` | Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) | -| dendrite_config.global.jetstream.disable_tls_validation | bool | `true` | Disables TLS validation. This should **NOT** be used in production. | -| dendrite_config.global.cache.max_size_estimated | string | `"1gb"` | The estimated maximum size for the global cache in bytes, or in terabytes, gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or 'kb' suffix is specified. Note that this is not a hard limit, nor is it a memory limit for the entire process. A cache that is too small may ultimately provide little or no benefit. | -| dendrite_config.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory before it will be evicted and/or refreshed from the database. Lower values result in easier admission of new cache entries but may also increase database load in comparison to higher values, so adjust conservatively. Higher values may make it harder for new items to make it into the cache, e.g. if new rooms suddenly become popular. | -| dendrite_config.global.report_stats.enabled | bool | `false` | Configures phone-home statistics reporting. These statistics contain the server name, number of active users and some information on your deployment config. We use this information to understand how Dendrite is being used in the wild. | -| dendrite_config.global.report_stats.endpoint | string | `"https://matrix.org/report-usage-stats/push"` | Endpoint to report statistics to. | -| dendrite_config.global.presence.enable_inbound | bool | `false` | Controls whether we receive presence events from other servers | -| dendrite_config.global.presence.enable_outbound | bool | `false` | Controls whether we send presence events for our local users to other servers. (_May increase CPU/memory usage_) | -| dendrite_config.global.server_notices.enabled | bool | `false` | Server notices allows server admins to send messages to all users on the server. | -| dendrite_config.global.server_notices.local_part | string | `"_server"` | The local part for the user sending server notices. | -| dendrite_config.global.server_notices.display_name | string | `"Server Alerts"` | The display name for the user sending server notices. | -| dendrite_config.global.server_notices.avatar_url | string | `""` | The avatar URL (as a mxc:// URL) name for the user sending server notices. | -| dendrite_config.global.server_notices.room_name | string | `"Server Alerts"` | | -| dendrite_config.global.metrics.enabled | bool | `false` | Whether or not Prometheus metrics are enabled. | -| dendrite_config.global.metrics.basic_auth.user | string | `"metrics"` | HTTP basic authentication username | -| dendrite_config.global.metrics.basic_auth.password | string | `"metrics"` | HTTP basic authentication password | -| dendrite_config.global.dns_cache.enabled | bool | `false` | Whether or not the DNS cache is enabled. | -| dendrite_config.global.dns_cache.cache_size | int | `256` | Maximum number of entries to hold in the DNS cache | -| dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | -| dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | -| dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | -| dendrite_config.mscs | object | `{"mscs":["msc2946"]}` | Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) | -| dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | -| dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | -| dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | -| dendrite_config.client_api.guests_disabled | bool | `true` | | -| dendrite_config.client_api.registration_shared_secret | string | `""` | If set, allows registration by anyone who knows the shared secret, regardless of whether registration is otherwise disabled. | -| dendrite_config.client_api.enable_registration_captcha | bool | `false` | enable reCAPTCHA registration | -| dendrite_config.client_api.recaptcha_public_key | string | `""` | reCAPTCHA public key | -| dendrite_config.client_api.recaptcha_private_key | string | `""` | reCAPTCHA private key | -| dendrite_config.client_api.recaptcha_bypass_secret | string | `""` | reCAPTCHA bypass secret | -| dendrite_config.client_api.recaptcha_siteverify_api | string | `""` | | -| dendrite_config.client_api.turn.turn_user_lifetime | string | `"24h"` | Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | -| dendrite_config.client_api.turn.turn_uris | list | `[]` | | -| dendrite_config.client_api.turn.turn_shared_secret | string | `""` | | -| dendrite_config.client_api.turn.turn_username | string | `""` | The TURN username | -| dendrite_config.client_api.turn.turn_password | string | `""` | The TURN password | -| dendrite_config.client_api.rate_limiting.enabled | bool | `true` | Enable rate limiting | -| dendrite_config.client_api.rate_limiting.threshold | int | `20` | After how many requests a rate limit should be activated | -| dendrite_config.client_api.rate_limiting.cooloff_ms | int | `500` | Cooloff time in milliseconds | -| dendrite_config.client_api.rate_limiting.exempt_user_ids | string | `nil` | Users which should be exempt from rate limiting | -| dendrite_config.federation_api.send_max_retries | int | `16` | Federation failure threshold. How many consecutive failures that we should tolerate when sending federation requests to a specific server. The backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. The default value is 16 if not specified, which is circa 18 hours. | -| dendrite_config.federation_api.disable_tls_validation | bool | `false` | Disable TLS validation. This should **NOT** be used in production. | -| dendrite_config.federation_api.prefer_direct_fetch | bool | `false` | | -| dendrite_config.federation_api.disable_http_keepalives | bool | `false` | Prevents Dendrite from keeping HTTP connections open for reuse for future requests. Connections will be closed quicker but we may spend more time on TLS handshakes instead. | -| dendrite_config.federation_api.key_perspectives | list | See value.yaml | Perspective keyservers, to use as a backup when direct key fetch requests don't succeed. | -| dendrite_config.media_api.base_path | string | `"/data/media_store"` | The path to store media files (e.g. avatars) in | -| dendrite_config.media_api.max_file_size_bytes | int | `10485760` | The max file size for uploaded media files | -| dendrite_config.media_api.dynamic_thumbnails | bool | `false` | | -| dendrite_config.media_api.max_thumbnail_generators | int | `10` | The maximum number of simultaneous thumbnail generators to run. | -| dendrite_config.media_api.thumbnail_sizes | list | See value.yaml | A list of thumbnail sizes to be generated for media content. | -| dendrite_config.sync_api.real_ip_header | string | `"X-Real-IP"` | This option controls which HTTP header to inspect to find the real remote IP address of the client. This is likely required if Dendrite is running behind a reverse proxy server. | -| dendrite_config.sync_api.search | object | `{"enabled":true,"index_path":"/data/search","language":"en"}` | Configuration for the full-text search engine. | -| dendrite_config.sync_api.search.enabled | bool | `true` | Whether fulltext search is enabled. | -| dendrite_config.sync_api.search.index_path | string | `"/data/search"` | The path to store the search index in. | -| dendrite_config.sync_api.search.language | string | `"en"` | The language most likely to be used on the server - used when indexing, to ensure the returned results match expectations. A full list of possible languages can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) | -| dendrite_config.user_api.bcrypt_cost | int | `10` | bcrypt cost to use when hashing passwords. (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) | -| dendrite_config.user_api.openid_token_lifetime_ms | int | `3600000` | OpenID Token lifetime in milliseconds. | -| dendrite_config.user_api.push_gateway_disable_tls_validation | bool | `false` | | -| dendrite_config.user_api.auto_join_rooms | list | `[]` | Rooms to join users to after registration | -| dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | -| postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | -| postgresql.image.repository | string | `"bitnami/postgresql"` | | -| postgresql.image.tag | string | `"15.1.0"` | | -| postgresql.auth.username | string | `"dendrite"` | | -| postgresql.auth.password | string | `"changeme"` | | -| postgresql.auth.database | string | `"dendrite"` | | -| postgresql.persistence.enabled | bool | `false` | | -| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | -| ingress.hosts | list | `[]` | | -| ingress.className | string | `""` | | -| ingress.hostName | string | `""` | | -| ingress.annotations | object | `{}` | Extra, custom annotations | -| ingress.tls | list | `[]` | | -| service.type | string | `"ClusterIP"` | | -| service.port | int | `8008` | | -| prometheus.servicemonitor.enabled | bool | `false` | Enable ServiceMonitor for Prometheus-Operator for scrape metric-endpoint | -| prometheus.servicemonitor.labels | object | `{}` | Extra Labels on ServiceMonitor for selector of Prometheus Instance | -| prometheus.rules.enabled | bool | `false` | Enable PrometheusRules for Prometheus-Operator for setup alerting | -| prometheus.rules.labels | object | `{}` | Extra Labels on PrometheusRules for selector of Prometheus Instance | -| prometheus.rules.additionalRules | list | `[]` | additional alertrules (no default alertrules are provided) | -| grafana.dashboards.enabled | bool | `false` | | -| grafana.dashboards.labels | object | `{"grafana_dashboard":"1"}` | Extra Labels on ConfigMap for selector of grafana sidecar | -| grafana.dashboards.annotations | object | `{}` | Extra Annotations on ConfigMap additional config in grafana sidecar | +| Key | Type | Default | Description | +| ------------------------------------------------------------ | ------ | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| image.repository | string | `"ghcr.io/matrix-org/dendrite-monolith"` | Docker repository/image to use | +| image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| signing_key.create | bool | `true` | Create a new signing key, if not exists | +| signing_key.existingSecret | string | `""` | Use an existing secret | +| resources | object | sets some sane default values | Default resource requests/limits. | +| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | +| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | +| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | +| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | +| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | +| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | +| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | +| extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | +| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | +| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | +| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | +| dendrite_config.version | int | `2` | | +| dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | +| dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | +| dendrite_config.global.well_known_server_name | string | `""` | The server name to delegate server-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.well_known_client_name | string | `""` | The server name to delegate client-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Lists of domains that the server will trust as identity servers to verify third party identifiers such as phone numbers and email addresses. | +| dendrite_config.global.old_private_keys | string | `nil` | The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) to old signing keys that were formerly in use on this domain name. These keys will not be used for federation request or event signing, but will be provided to any other homeserver that asks when trying to verify old events. | +| dendrite_config.global.disable_federation | bool | `false` | Disable federation. Dendrite will not be able to make any outbound HTTP requests to other servers and the federation API will not be exposed. | +| dendrite_config.global.key_validity_period | string | `"168h0m0s"` | | +| dendrite_config.global.database.connection_string | string | `""` | The connection string for connections to Postgres. This will be set automatically if using the Postgres dependency | +| dendrite_config.global.database.max_open_conns | int | `90` | Default database maximum open connections | +| dendrite_config.global.database.max_idle_conns | int | `5` | Default database maximum idle connections | +| dendrite_config.global.database.conn_max_lifetime | int | `-1` | Default database maximum lifetime | +| dendrite_config.global.jetstream.storage_path | string | `"/data/jetstream"` | Persistent directory to store JetStream streams in. | +| dendrite_config.global.jetstream.addresses | list | `[]` | NATS JetStream server addresses if not using internal NATS. | +| dendrite_config.global.jetstream.topic_prefix | string | `"Dendrite"` | The prefix for JetStream streams | +| dendrite_config.global.jetstream.in_memory | bool | `false` | Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) | +| dendrite_config.global.jetstream.disable_tls_validation | bool | `true` | Disables TLS validation. This should **NOT** be used in production. | +| dendrite_config.global.cache.max_size_estimated | string | `"1gb"` | The estimated maximum size for the global cache in bytes, or in terabytes, gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or 'kb' suffix is specified. Note that this is not a hard limit, nor is it a memory limit for the entire process. A cache that is too small may ultimately provide little or no benefit. | +| dendrite_config.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory before it will be evicted and/or refreshed from the database. Lower values result in easier admission of new cache entries but may also increase database load in comparison to higher values, so adjust conservatively. Higher values may make it harder for new items to make it into the cache, e.g. if new rooms suddenly become popular. | +| dendrite_config.global.report_stats.enabled | bool | `false` | Configures phone-home statistics reporting. These statistics contain the server name, number of active users and some information on your deployment config. We use this information to understand how Dendrite is being used in the wild. | +| dendrite_config.global.report_stats.endpoint | string | `"https://matrix.org/report-usage-stats/push"` | Endpoint to report statistics to. | +| dendrite_config.global.presence.enable_inbound | bool | `false` | Controls whether we receive presence events from other servers | +| dendrite_config.global.presence.enable_outbound | bool | `false` | Controls whether we send presence events for our local users to other servers. (_May increase CPU/memory usage_) | +| dendrite_config.global.server_notices.enabled | bool | `false` | Server notices allows server admins to send messages to all users on the server. | +| dendrite_config.global.server_notices.local_part | string | `"_server"` | The local part for the user sending server notices. | +| dendrite_config.global.server_notices.display_name | string | `"Server Alerts"` | The display name for the user sending server notices. | +| dendrite_config.global.server_notices.avatar_url | string | `""` | The avatar URL (as a mxc:// URL) name for the user sending server notices. | +| dendrite_config.global.server_notices.room_name | string | `"Server Alerts"` | | +| dendrite_config.global.metrics.enabled | bool | `false` | Whether or not Prometheus metrics are enabled. | +| dendrite_config.global.metrics.basic_auth.user | string | `"metrics"` | HTTP basic authentication username | +| dendrite_config.global.metrics.basic_auth.password | string | `"metrics"` | HTTP basic authentication password | +| dendrite_config.global.dns_cache.enabled | bool | `false` | Whether or not the DNS cache is enabled. | +| dendrite_config.global.dns_cache.cache_size | int | `256` | Maximum number of entries to hold in the DNS cache | +| dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | +| dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | +| dendrite_config.mscs | object | `{"mscs":["msc2836"]}` | Configuration for experimental MSC's. (Valid values are: msc2836) | +| dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | +| dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | +| dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | +| dendrite_config.client_api.guests_disabled | bool | `true` | | +| dendrite_config.client_api.registration_shared_secret | string | `""` | If set, allows registration by anyone who knows the shared secret, regardless of whether registration is otherwise disabled. | +| dendrite_config.client_api.enable_registration_captcha | bool | `false` | enable reCAPTCHA registration | +| dendrite_config.client_api.recaptcha_public_key | string | `""` | reCAPTCHA public key | +| dendrite_config.client_api.recaptcha_private_key | string | `""` | reCAPTCHA private key | +| dendrite_config.client_api.recaptcha_bypass_secret | string | `""` | reCAPTCHA bypass secret | +| dendrite_config.client_api.recaptcha_siteverify_api | string | `""` | | +| dendrite_config.client_api.turn.turn_user_lifetime | string | `"24h"` | Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.client_api.turn.turn_uris | list | `[]` | | +| dendrite_config.client_api.turn.turn_shared_secret | string | `""` | | +| dendrite_config.client_api.turn.turn_username | string | `""` | The TURN username | +| dendrite_config.client_api.turn.turn_password | string | `""` | The TURN password | +| dendrite_config.client_api.rate_limiting.enabled | bool | `true` | Enable rate limiting | +| dendrite_config.client_api.rate_limiting.threshold | int | `20` | After how many requests a rate limit should be activated | +| dendrite_config.client_api.rate_limiting.cooloff_ms | int | `500` | Cooloff time in milliseconds | +| dendrite_config.client_api.rate_limiting.exempt_user_ids | string | `nil` | Users which should be exempt from rate limiting | +| dendrite_config.federation_api.send_max_retries | int | `16` | Federation failure threshold. How many consecutive failures that we should tolerate when sending federation requests to a specific server. The backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. The default value is 16 if not specified, which is circa 18 hours. | +| dendrite_config.federation_api.disable_tls_validation | bool | `false` | Disable TLS validation. This should **NOT** be used in production. | +| dendrite_config.federation_api.prefer_direct_fetch | bool | `false` | | +| dendrite_config.federation_api.disable_http_keepalives | bool | `false` | Prevents Dendrite from keeping HTTP connections open for reuse for future requests. Connections will be closed quicker but we may spend more time on TLS handshakes instead. | +| dendrite_config.federation_api.key_perspectives | list | See value.yaml | Perspective keyservers, to use as a backup when direct key fetch requests don't succeed. | +| dendrite_config.media_api.base_path | string | `"/data/media_store"` | The path to store media files (e.g. avatars) in | +| dendrite_config.media_api.max_file_size_bytes | int | `10485760` | The max file size for uploaded media files | +| dendrite_config.media_api.dynamic_thumbnails | bool | `false` | | +| dendrite_config.media_api.max_thumbnail_generators | int | `10` | The maximum number of simultaneous thumbnail generators to run. | +| dendrite_config.media_api.thumbnail_sizes | list | See value.yaml | A list of thumbnail sizes to be generated for media content. | +| dendrite_config.sync_api.real_ip_header | string | `"X-Real-IP"` | This option controls which HTTP header to inspect to find the real remote IP address of the client. This is likely required if Dendrite is running behind a reverse proxy server. | +| dendrite_config.sync_api.search | object | `{"enabled":true,"index_path":"/data/search","language":"en"}` | Configuration for the full-text search engine. | +| dendrite_config.sync_api.search.enabled | bool | `true` | Whether fulltext search is enabled. | +| dendrite_config.sync_api.search.index_path | string | `"/data/search"` | The path to store the search index in. | +| dendrite_config.sync_api.search.language | string | `"en"` | The language most likely to be used on the server - used when indexing, to ensure the returned results match expectations. A full list of possible languages can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) | +| dendrite_config.user_api.bcrypt_cost | int | `10` | bcrypt cost to use when hashing passwords. (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) | +| dendrite_config.user_api.openid_token_lifetime_ms | int | `3600000` | OpenID Token lifetime in milliseconds. | +| dendrite_config.user_api.push_gateway_disable_tls_validation | bool | `false` | | +| dendrite_config.user_api.auto_join_rooms | list | `[]` | Rooms to join users to after registration | +| dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | +| postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | +| postgresql.image.repository | string | `"bitnami/postgresql"` | | +| postgresql.image.tag | string | `"15.1.0"` | | +| postgresql.auth.username | string | `"dendrite"` | | +| postgresql.auth.password | string | `"changeme"` | | +| postgresql.auth.database | string | `"dendrite"` | | +| postgresql.persistence.enabled | bool | `false` | | +| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | +| ingress.hosts | list | `[]` | | +| ingress.className | string | `""` | | +| ingress.hostName | string | `""` | | +| ingress.annotations | object | `{}` | Extra, custom annotations | +| ingress.tls | list | `[]` | | +| service.type | string | `"ClusterIP"` | | +| service.port | int | `8008` | | +| prometheus.servicemonitor.enabled | bool | `false` | Enable ServiceMonitor for Prometheus-Operator for scrape metric-endpoint | +| prometheus.servicemonitor.labels | object | `{}` | Extra Labels on ServiceMonitor for selector of Prometheus Instance | +| prometheus.rules.enabled | bool | `false` | Enable PrometheusRules for Prometheus-Operator for setup alerting | +| prometheus.rules.labels | object | `{}` | Extra Labels on PrometheusRules for selector of Prometheus Instance | +| prometheus.rules.additionalRules | list | `[]` | additional alertrules (no default alertrules are provided) | +| grafana.dashboards.enabled | bool | `false` | | +| grafana.dashboards.labels | object | `{"grafana_dashboard":"1"}` | Extra Labels on ConfigMap for selector of grafana sidecar | +| grafana.dashboards.annotations | object | `{}` | Extra Annotations on ConfigMap additional config in grafana sidecar | ## Monitoring diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index 2b009c7d64..e3ad0f963f 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -211,14 +211,12 @@ dendrite_config: # -- pprof port, if enabled port: 65432 - # -- Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) + # -- Configuration for experimental MSC's. (Valid values are: msc2836) mscs: mscs: - - msc2946 # A list of enabled MSC's # Currently valid values are: # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) app_service_api: # -- Disable the validation of TLS certificates of appservices. This is diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index 21d4b4da04..ce491cd724 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -7,7 +7,6 @@ type MSCs struct { // 'msc2444': Peeking over federation - https://github.com/matrix-org/matrix-doc/pull/2444 // 'msc2753': Peeking via /sync - https://github.com/matrix-org/matrix-doc/pull/2753 // 'msc2836': Threading - https://github.com/matrix-org/matrix-doc/pull/2836 - // 'msc2946': Spaces Summary - https://github.com/matrix-org/matrix-doc/pull/2946 MSCs []string `yaml:"mscs"` Database DatabaseOptions `yaml:"database,omitempty"` diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go deleted file mode 100644 index 3a0c181dc1..0000000000 --- a/setup/mscs/msc2946/msc2946.go +++ /dev/null @@ -1,744 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package msc2946 'Spaces Summary' implements https://github.com/matrix-org/matrix-doc/pull/2946 -package msc2946 - -import ( - "context" - "encoding/json" - "net/http" - "net/url" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/google/uuid" - "github.com/gorilla/mux" - fs "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httputil" - roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" - "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/util" - "github.com/tidwall/gjson" -) - -const ( - ConstCreateEventContentKey = "type" - ConstCreateEventContentValueSpace = "m.space" - ConstSpaceChildEventType = "m.space.child" - ConstSpaceParentEventType = "m.space.parent" -) - -type MSC2946ClientResponse struct { - Rooms []fclient.MSC2946Room `json:"rooms"` - NextBatch string `json:"next_batch,omitempty"` -} - -// Enable this MSC -func Enable( - cfg *config.Dendrite, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, - fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, cache caching.RoomHierarchyCache, -) error { - clientAPI := httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(rsAPI, fsAPI, cache, cfg.Global.ServerName), httputil.WithAllowGuests()) - routers.Client.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) - routers.Client.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) - - fedAPI := httputil.MakeExternalAPI( - "msc2946_fed_spaces", func(req *http.Request) util.JSONResponse { - fedReq, errResp := fclient.VerifyHTTPRequest( - req, time.Now(), cfg.Global.ServerName, cfg.Global.IsLocalServerName, keyRing, - ) - if fedReq == nil { - return errResp - } - // Extract the room ID from the request. Sanity check request data. - params, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - roomID := params["roomID"] - return federatedSpacesHandler(req.Context(), fedReq, roomID, cache, rsAPI, fsAPI, cfg.Global.ServerName) - }, - ) - routers.Federation.Handle("/unstable/org.matrix.msc2946/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) - routers.Federation.Handle("/v1/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) - return nil -} - -func federatedSpacesHandler( - ctx context.Context, fedReq *fclient.FederationRequest, roomID string, - cache caching.RoomHierarchyCache, - rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, - thisServer spec.ServerName, -) util.JSONResponse { - u, err := url.Parse(fedReq.RequestURI()) - if err != nil { - return util.JSONResponse{ - Code: 400, - JSON: spec.InvalidParam("bad request uri"), - } - } - - w := walker{ - rootRoomID: roomID, - serverName: fedReq.Origin(), - thisServer: thisServer, - ctx: ctx, - cache: cache, - suggestedOnly: u.Query().Get("suggested_only") == "true", - limit: 1000, - // The main difference is that it does not recurse into spaces and does not support pagination. - // This is somewhat equivalent to a Client-Server request with a max_depth=1. - maxDepth: 1, - - rsAPI: rsAPI, - fsAPI: fsAPI, - // inline cache as we don't have pagination in federation mode - paginationCache: make(map[string]paginationInfo), - } - return w.walk() -} - -func spacesHandler( - rsAPI roomserver.RoomserverInternalAPI, - fsAPI fs.FederationInternalAPI, - cache caching.RoomHierarchyCache, - thisServer spec.ServerName, -) func(*http.Request, *userapi.Device) util.JSONResponse { - // declared outside the returned handler so it persists between calls - // TODO: clear based on... time? - paginationCache := make(map[string]paginationInfo) - - return func(req *http.Request, device *userapi.Device) util.JSONResponse { - // Extract the room ID from the request. Sanity check request data. - params, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - roomID := params["roomID"] - w := walker{ - suggestedOnly: req.URL.Query().Get("suggested_only") == "true", - limit: parseInt(req.URL.Query().Get("limit"), 1000), - maxDepth: parseInt(req.URL.Query().Get("max_depth"), -1), - paginationToken: req.URL.Query().Get("from"), - rootRoomID: roomID, - caller: device, - thisServer: thisServer, - ctx: req.Context(), - cache: cache, - - rsAPI: rsAPI, - fsAPI: fsAPI, - paginationCache: paginationCache, - } - return w.walk() - } -} - -type paginationInfo struct { - processed set - unvisited []roomVisit -} - -type walker struct { - rootRoomID string - caller *userapi.Device - serverName spec.ServerName - thisServer spec.ServerName - rsAPI roomserver.RoomserverInternalAPI - fsAPI fs.FederationInternalAPI - ctx context.Context - cache caching.RoomHierarchyCache - suggestedOnly bool - limit int - maxDepth int - paginationToken string - - paginationCache map[string]paginationInfo - mu sync.Mutex -} - -func (w *walker) newPaginationCache() (string, paginationInfo) { - p := paginationInfo{ - processed: make(set), - unvisited: nil, - } - tok := uuid.NewString() - return tok, p -} - -func (w *walker) loadPaginationCache(paginationToken string) *paginationInfo { - w.mu.Lock() - defer w.mu.Unlock() - p := w.paginationCache[paginationToken] - return &p -} - -func (w *walker) storePaginationCache(paginationToken string, cache paginationInfo) { - w.mu.Lock() - defer w.mu.Unlock() - w.paginationCache[paginationToken] = cache -} - -type roomVisit struct { - roomID string - parentRoomID string - depth int - vias []string // vias to query this room by -} - -func (w *walker) walk() util.JSONResponse { - if authorised, _ := w.authorised(w.rootRoomID, ""); !authorised { - if w.caller != nil { - // CS API format - return util.JSONResponse{ - Code: 403, - JSON: spec.Forbidden("room is unknown/forbidden"), - } - } else { - // SS API format - return util.JSONResponse{ - Code: 404, - JSON: spec.NotFound("room is unknown/forbidden"), - } - } - } - - var discoveredRooms []fclient.MSC2946Room - - var cache *paginationInfo - if w.paginationToken != "" { - cache = w.loadPaginationCache(w.paginationToken) - if cache == nil { - return util.JSONResponse{ - Code: 400, - JSON: spec.InvalidParam("invalid from"), - } - } - } else { - tok, c := w.newPaginationCache() - cache = &c - w.paginationToken = tok - // Begin walking the graph starting with the room ID in the request in a queue of unvisited rooms - c.unvisited = append(c.unvisited, roomVisit{ - roomID: w.rootRoomID, - parentRoomID: "", - depth: 0, - }) - } - - processed := cache.processed - unvisited := cache.unvisited - - // Depth first -> stack data structure - for len(unvisited) > 0 { - if len(discoveredRooms) >= w.limit { - break - } - - // pop the stack - rv := unvisited[len(unvisited)-1] - unvisited = unvisited[:len(unvisited)-1] - // If this room has already been processed, skip. - // If this room exceeds the specified depth, skip. - if processed.isSet(rv.roomID) || rv.roomID == "" || (w.maxDepth > 0 && rv.depth > w.maxDepth) { - continue - } - - // Mark this room as processed. - processed.set(rv.roomID) - - // if this room is not a space room, skip. - var roomType string - create := w.stateEvent(rv.roomID, spec.MRoomCreate, "") - if create != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str - } - - // Collect rooms/events to send back (either locally or fetched via federation) - var discoveredChildEvents []fclient.MSC2946StrippedEvent - - // If we know about this room and the caller is authorised (joined/world_readable) then pull - // events locally - roomExists := w.roomExists(rv.roomID) - if !roomExists { - // attempt to query this room over federation, as either we've never heard of it before - // or we've left it and hence are not authorised (but info may be exposed regardless) - fedRes := w.federatedRoomInfo(rv.roomID, rv.vias) - if fedRes != nil { - discoveredChildEvents = fedRes.Room.ChildrenState - discoveredRooms = append(discoveredRooms, fedRes.Room) - if len(fedRes.Children) > 0 { - discoveredRooms = append(discoveredRooms, fedRes.Children...) - } - // mark this room as a space room as the federated server responded. - // we need to do this so we add the children of this room to the unvisited stack - // as these children may be rooms we do know about. - roomType = ConstCreateEventContentValueSpace - } - } else if authorised, isJoinedOrInvited := w.authorised(rv.roomID, rv.parentRoomID); authorised { - // Get all `m.space.child` state events for this room - events, err := w.childReferences(rv.roomID) - if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("room_id", rv.roomID).Error("failed to extract references for room") - continue - } - discoveredChildEvents = events - - pubRoom := w.publicRoomsChunk(rv.roomID) - - discoveredRooms = append(discoveredRooms, fclient.MSC2946Room{ - PublicRoom: *pubRoom, - RoomType: roomType, - ChildrenState: events, - }) - // don't walk children if the user is not joined/invited to the space - if !isJoinedOrInvited { - continue - } - } else { - // room exists but user is not authorised - continue - } - - // don't walk the children - // if the parent is not a space room - if roomType != ConstCreateEventContentValueSpace { - continue - } - - // For each referenced room ID in the child events being returned to the caller - // add the room ID to the queue of unvisited rooms. Loop from the beginning. - // We need to invert the order here because the child events are lo->hi on the timestamp, - // so we need to ensure we pop in the same lo->hi order, which won't be the case if we - // insert the highest timestamp last in a stack. - for i := len(discoveredChildEvents) - 1; i >= 0; i-- { - spaceContent := struct { - Via []string `json:"via"` - }{} - ev := discoveredChildEvents[i] - _ = json.Unmarshal(ev.Content, &spaceContent) - unvisited = append(unvisited, roomVisit{ - roomID: ev.StateKey, - parentRoomID: rv.roomID, - depth: rv.depth + 1, - vias: spaceContent.Via, - }) - } - } - - if len(unvisited) > 0 { - // we still have more rooms so we need to send back a pagination token, - // we probably hit a room limit - cache.processed = processed - cache.unvisited = unvisited - w.storePaginationCache(w.paginationToken, *cache) - } else { - // clear the pagination token so we don't send it back to the client - // Note we do NOT nuke the cache just in case this response is lost - // and the client retries it. - w.paginationToken = "" - } - - if w.caller != nil { - // return CS API format - return util.JSONResponse{ - Code: 200, - JSON: MSC2946ClientResponse{ - Rooms: discoveredRooms, - NextBatch: w.paginationToken, - }, - } - } - // return SS API format - // the first discovered room will be the room asked for, and subsequent ones the depth=1 children - if len(discoveredRooms) == 0 { - return util.JSONResponse{ - Code: 404, - JSON: spec.NotFound("room is unknown/forbidden"), - } - } - return util.JSONResponse{ - Code: 200, - JSON: fclient.MSC2946SpacesResponse{ - Room: discoveredRooms[0], - Children: discoveredRooms[1:], - }, - } -} - -func (w *walker) stateEvent(roomID, evType, stateKey string) *types.HeaderedEvent { - var queryRes roomserver.QueryCurrentStateResponse - tuple := gomatrixserverlib.StateKeyTuple{ - EventType: evType, - StateKey: stateKey, - } - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, - }, &queryRes) - if err != nil { - return nil - } - return queryRes.StateEvents[tuple] -} - -func (w *walker) publicRoomsChunk(roomID string) *fclient.PublicRoom { - pubRooms, err := roomserver.PopulatePublicRooms(w.ctx, []string{roomID}, w.rsAPI) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to PopulatePublicRooms") - return nil - } - if len(pubRooms) == 0 { - return nil - } - return &pubRooms[0] -} - -// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was -// unsuccessful. -func (w *walker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC2946SpacesResponse { - // only do federated requests for client requests - if w.caller == nil { - return nil - } - resp, ok := w.cache.GetRoomHierarchy(roomID) - if ok { - util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID) - return &resp - } - util.GetLogger(w.ctx).Debugf("Querying %s via %+v", roomID, vias) - ctx := context.Background() - // query more of the spaces graph using these servers - for _, serverName := range vias { - if serverName == string(w.thisServer) { - continue - } - res, err := w.fsAPI.RoomHierarchies(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) - continue - } - // ensure nil slices are empty as we send this to the client sometimes - if res.Room.ChildrenState == nil { - res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} - } - for i := 0; i < len(res.Children); i++ { - child := res.Children[i] - if child.ChildrenState == nil { - child.ChildrenState = []fclient.MSC2946StrippedEvent{} - } - res.Children[i] = child - } - w.cache.StoreRoomHierarchy(roomID, res) - - return &res - } - return nil -} - -func (w *walker) roomExists(roomID string) bool { - var queryRes roomserver.QueryServerJoinedToRoomResponse - err := w.rsAPI.QueryServerJoinedToRoom(w.ctx, &roomserver.QueryServerJoinedToRoomRequest{ - RoomID: roomID, - ServerName: w.thisServer, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") - return false - } - // if the room exists but we aren't in the room then we might have stale data so we want to fetch - // it fresh via federation - return queryRes.RoomExists && queryRes.IsInRoom -} - -// authorised returns true iff the user is joined this room or the room is world_readable -func (w *walker) authorised(roomID, parentRoomID string) (authed, isJoinedOrInvited bool) { - if w.caller != nil { - return w.authorisedUser(roomID, parentRoomID) - } - return w.authorisedServer(roomID), false -} - -// authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable -func (w *walker) authorisedServer(roomID string) bool { - // Check history visibility / join rules first - hisVisTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomHistoryVisibility, - StateKey: "", - } - joinRuleTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomJoinRules, - StateKey: "", - } - var queryRoomRes roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - hisVisTuple, joinRuleTuple, - }, - }, &queryRoomRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") - return false - } - hisVisEv := queryRoomRes.StateEvents[hisVisTuple] - if hisVisEv != nil { - hisVis, _ := hisVisEv.HistoryVisibility() - if hisVis == "world_readable" { - return true - } - } - - // check if this room is a restricted room and if so, we need to check if the server is joined to an allowed room ID - // in addition to the actual room ID (but always do the actual one first as it's quicker in the common case) - allowJoinedToRoomIDs := []string{roomID} - joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple] - - if joinRuleEv != nil { - rule, ruleErr := joinRuleEv.JoinRule() - if ruleErr != nil { - util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") - return false - } - - if rule == spec.Public || rule == spec.Knock { - return true - } - - if rule == spec.Restricted { - allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...) - } - } - - // check if server is joined to any allowed room - for _, allowedRoomID := range allowJoinedToRoomIDs { - var queryRes fs.QueryJoinedHostServerNamesInRoomResponse - err = w.fsAPI.QueryJoinedHostServerNamesInRoom(w.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: allowedRoomID, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") - continue - } - for _, srv := range queryRes.ServerNames { - if srv == w.serverName { - return true - } - } - } - - return false -} - -// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable -// or if the room has a public or knock join rule. -// Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true. -func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoinedOrInvited bool) { - hisVisTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomHistoryVisibility, - StateKey: "", - } - joinRuleTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomJoinRules, - StateKey: "", - } - roomMemberTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomMember, - StateKey: w.caller.UserID, - } - var queryRes roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - hisVisTuple, joinRuleTuple, roomMemberTuple, - }, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") - return false, false - } - memberEv := queryRes.StateEvents[roomMemberTuple] - if memberEv != nil { - membership, _ := memberEv.Membership() - if membership == spec.Join || membership == spec.Invite { - return true, true - } - } - hisVisEv := queryRes.StateEvents[hisVisTuple] - if hisVisEv != nil { - hisVis, _ := hisVisEv.HistoryVisibility() - if hisVis == "world_readable" { - return true, false - } - } - joinRuleEv := queryRes.StateEvents[joinRuleTuple] - if parentRoomID != "" && joinRuleEv != nil { - var allowed bool - rule, ruleErr := joinRuleEv.JoinRule() - if ruleErr != nil { - util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule") - } else if rule == spec.Public || rule == spec.Knock { - allowed = true - } else if rule == spec.Restricted { - allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership") - // check parent is in the allowed set - for _, a := range allowedRoomIDs { - if parentRoomID == a { - allowed = true - break - } - } - } - if allowed { - // ensure caller is joined to the parent room - var queryRes2 roomserver.QueryCurrentStateResponse - err = w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: parentRoomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - roomMemberTuple, - }, - }, &queryRes2) - if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("parent_room_id", parentRoomID).Warn("failed to check user is joined to parent room") - } else { - memberEv = queryRes2.StateEvents[roomMemberTuple] - if memberEv != nil { - membership, _ := memberEv.Membership() - if membership == spec.Join { - return true, false - } - } - } - } - } - return false, false -} - -func (w *walker) restrictedJoinRuleAllowedRooms(joinRuleEv *types.HeaderedEvent, allowType string) (allows []string) { - rule, _ := joinRuleEv.JoinRule() - if rule != spec.Restricted { - return nil - } - var jrContent gomatrixserverlib.JoinRuleContent - if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil { - util.GetLogger(w.ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err) - return nil - } - for _, allow := range jrContent.Allow { - if allow.Type == allowType { - allows = append(allows, allow.RoomID) - } - } - return -} - -// references returns all child references pointing to or from this room. -func (w *walker) childReferences(roomID string) ([]fclient.MSC2946StrippedEvent, error) { - createTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomCreate, - StateKey: "", - } - var res roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(context.Background(), &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - AllowWildcards: true, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - createTuple, { - EventType: ConstSpaceChildEventType, - StateKey: "*", - }, - }, - }, &res) - if err != nil { - return nil, err - } - - // don't return any child refs if the room is not a space room - if res.StateEvents[createTuple] != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str - if roomType != ConstCreateEventContentValueSpace { - return []fclient.MSC2946StrippedEvent{}, nil - } - } - delete(res.StateEvents, createTuple) - - el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) - for _, ev := range res.StateEvents { - content := gjson.ParseBytes(ev.Content()) - // only return events that have a `via` key as per MSC1772 - // else we'll incorrectly walk redacted events (as the link - // is in the state_key) - if content.Get("via").Exists() { - strip := stripped(ev.PDU) - if strip == nil { - continue - } - // if suggested only and this child isn't suggested, skip it. - // if suggested only = false we include everything so don't need to check the content. - if w.suggestedOnly && !content.Get("suggested").Bool() { - continue - } - el = append(el, *strip) - } - } - // sort by origin_server_ts as per MSC2946 - sort.Slice(el, func(i, j int) bool { - return el[i].OriginServerTS < el[j].OriginServerTS - }) - - return el, nil -} - -type set map[string]struct{} - -func (s set) set(val string) { - s[val] = struct{}{} -} -func (s set) isSet(val string) bool { - _, ok := s[val] - return ok -} - -func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { - if ev.StateKey() == nil { - return nil - } - return &fclient.MSC2946StrippedEvent{ - Type: ev.Type(), - StateKey: *ev.StateKey(), - Content: ev.Content(), - Sender: string(ev.SenderID()), - OriginServerTS: ev.OriginServerTS(), - } -} - -func parseInt(intstr string, defaultVal int) int { - i, err := strconv.ParseInt(intstr, 10, 32) - if err != nil { - return defaultVal - } - return int(i) -} diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index 9cd5eed1cd..2084be2d0b 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -25,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs/msc2836" - "github.com/matrix-org/dendrite/setup/mscs/msc2946" "github.com/matrix-org/util" ) @@ -44,8 +43,6 @@ func EnableMSC(cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Ro switch msc { case "msc2836": return msc2836.Enable(cfg, cm, routers, monolith.RoomserverAPI, monolith.FederationAPI, monolith.UserAPI, monolith.KeyRing) - case "msc2946": - return msc2946.Enable(cfg, routers, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing, caches) case "msc2444": // enabled inside federationapi case "msc2753": // enabled inside clientapi default: From 19d420e3b2ad05ca5ecfafdd31507ad9da0d0eb6 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Mon, 17 Jul 2023 13:17:57 +0100 Subject: [PATCH 14/21] fix 2 bugs - CSAPI used wrong function to create pagination cache (which caused nil map errs) - Copied an array wrong in room hierarchy impl, which caused a nil room list to return --- clientapi/routing/routing.go | 4 ++-- roomserver/internal/query/query_room_hierarchy.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5e691e8ec5..8cd207b7a0 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -509,14 +509,14 @@ func Setup( // Defined outside of handler to persist between calls // TODO: clear based on some criteria - roomHierarchyPaginationCache := new(RoomHierarchyPaginationCache) + roomHierarchyPaginationCache := NewRoomHierarchyPaginationCache() v1mux.Handle("/rooms/{roomID}/hierarchy", httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, roomHierarchyPaginationCache) + return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, &roomHierarchyPaginationCache) }, httputil.WithAllowGuests()), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index d10cae25e4..a59c834d6a 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -43,10 +43,10 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} } - var discoveredRooms []fclient.MSC2946Room + discoveredRooms := []fclient.MSC2946Room{} - // Copy unvisited and processed to avoid modifying walker - unvisited := []roomserver.RoomHierarchyWalkerQueuedRoom{} + // Copy unvisited and processed to avoid modifying original walker (which is typically in cache) + unvisited := make([]roomserver.RoomHierarchyWalkerQueuedRoom, len(walker.Unvisited)) copy(unvisited, walker.Unvisited) processed := walker.Processed.Copy() From 14d4043b418fd4a43185f86962d247c6fe16dbdd Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Tue, 18 Jul 2023 15:42:08 +0100 Subject: [PATCH 15/21] add a few debug logs --- clientapi/routing/room_hierarchy.go | 1 + federationapi/routing/query.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index e314d07c82..b3c8e5900a 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -143,6 +143,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str if err != nil { switch err.(type) { case roomserverAPI.ErrRoomUnknownOrNotAllowed: + util.GetLogger(req.Context()).WithError(err).Debugln("room unknown/forbidden when handling CS room hierarchy request") return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("room is unknown/forbidden"), diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 8b37fbe7bf..f7957b3de8 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -151,6 +151,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques if err != nil { switch err.(type) { case roomserverAPI.ErrRoomUnknownOrNotAllowed: + util.GetLogger(httpReq.Context()).WithError(err).Debugln("room unknown/forbidden when handling SS room hierarchy request") return util.JSONResponse{ Code: http.StatusNotFound, JSON: spec.NotFound("room is unknown/forbidden"), @@ -165,6 +166,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques } if len(discoveredRooms) == 0 { + util.GetLogger(httpReq.Context()).Debugln("no rooms found when handling SS room hierarchy request") return util.JSONResponse{ Code: 404, JSON: spec.NotFound("room is unknown/forbidden"), From 0c7671dddb6f4e731e5e8bc6897c550cc0c084a1 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Tue, 18 Jul 2023 15:43:05 +0100 Subject: [PATCH 16/21] fix bug where limit = -1 returning nothing - internal room hierarchy function should treat -1 as no limit, it wasnt --- roomserver/internal/query/query_room_hierarchy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index a59c834d6a..b05db74291 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -52,7 +52,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r // Depth first -> stack data structure for len(unvisited) > 0 { - if len(discoveredRooms) >= limit { + if len(discoveredRooms) >= limit && limit != -1 { break } From 98033370754fe94d06bb7f6bfdbf4a458bf8b509 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Wed, 19 Jul 2023 12:59:31 +0100 Subject: [PATCH 17/21] update faq --- docs/FAQ.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/FAQ.md b/docs/FAQ.md index 0273d53823..570ba677e9 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -64,6 +64,10 @@ Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially suppo ## Does Dendrite support Space Summaries? +Yes + +## Does Dendrite support Threads? + Yes, to enable them [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. ``` From 46d558db290c2213c993bfbaf5ab173f2220411d Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Wed, 19 Jul 2023 13:07:04 +0100 Subject: [PATCH 18/21] clarify initialisation of APIs order in comments --- roomserver/roomserver.go | 3 +++ userapi/userapi.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 4685f474f3..65523e3c2d 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -28,6 +28,9 @@ import ( ) // NewInternalAPI returns a concrete implementation of the internal API. +// +// Many of the methods provided by this API depend on access to a federation API, and so +// you may wish to call `SetFederationAPI` on the returned struct to avoid nil-dereference errors. func NewInternalAPI( processContext *process.ProcessContext, cfg *config.Dendrite, diff --git a/userapi/userapi.go b/userapi/userapi.go index 6dcbc121f5..c47c6f0011 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -36,6 +36,9 @@ import ( // NewInternalAPI returns a concrete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +// +// Creating a new instance of the user API requires a roomserver API with a federation API set +// using its `SetFederationAPI` method, other you may get nil-dereference errors. func NewInternalAPI( processContext *process.ProcessContext, dendriteCfg *config.Dendrite, From 51aafae840d03594e9d9644412fc3a5eecd8437b Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Wed, 19 Jul 2023 13:35:15 +0100 Subject: [PATCH 19/21] update GMSL symbols - to use new symbols from https://github.com/matrix-org/gomatrixserverlib/pull/403 that don't mention MSC2946 --- clientapi/routing/room_hierarchy.go | 4 ++-- federationapi/api/api.go | 2 +- federationapi/internal/federationclient.go | 6 ++--- federationapi/routing/query.go | 2 +- go.mod | 2 +- go.sum | 10 ++------ internal/caching/cache_space_rooms.go | 8 +++---- internal/caching/caches.go | 2 +- internal/caching/impl_ristretto.go | 2 +- roomserver/api/api.go | 2 +- .../internal/query/query_room_hierarchy.go | 24 +++++++++---------- 11 files changed, 29 insertions(+), 35 deletions(-) diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index b3c8e5900a..2884d2c32d 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -175,6 +175,6 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str // Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy type RoomHierarchyClientResponse struct { - Rooms []fclient.MSC2946Room `json:"rooms"` - NextBatch string `json:"next_batch,omitempty"` + Rooms []fclient.RoomHierarchyRoom `json:"rooms"` + NextBatch string `json:"next_batch,omitempty"` } diff --git a/federationapi/api/api.go b/federationapi/api/api.go index 38cd393758..efe0547df0 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -75,7 +75,7 @@ type RoomserverFederationAPI interface { GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error) - RoomHierarchies(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) + RoomHierarchies(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.RoomHierarchyResponse, err error) } type P2PFederationAPI interface { diff --git a/federationapi/internal/federationclient.go b/federationapi/internal/federationclient.go index d5a45248d9..13270f6a28 100644 --- a/federationapi/internal/federationclient.go +++ b/federationapi/internal/federationclient.go @@ -196,14 +196,14 @@ func (a *FederationInternalAPI) MSC2836EventRelationships( func (a *FederationInternalAPI) RoomHierarchies( ctx context.Context, origin, s spec.ServerName, roomID string, suggestedOnly bool, -) (res fclient.MSC2946SpacesResponse, err error) { +) (res fclient.RoomHierarchyResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { - return a.federation.MSC2946Spaces(ctx, origin, s, roomID, suggestedOnly) + return a.federation.RoomHierarchy(ctx, origin, s, roomID, suggestedOnly) }) if err != nil { return res, err } - return ires.(fclient.MSC2946SpacesResponse), nil + return ires.(fclient.RoomHierarchyResponse), nil } diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index f7957b3de8..327ba9b08a 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -174,7 +174,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques } return util.JSONResponse{ Code: 200, - JSON: fclient.MSC2946SpacesResponse{ + JSON: fclient.RoomHierarchyResponse{ Room: discoveredRooms[0], Children: discoveredRooms[1:], }, diff --git a/go.mod b/go.mod index 7072ec81d3..c936b41515 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230713175830-cf82e01c0017 + github.com/matrix-org/gomatrixserverlib v0.0.0-20230719122259-2bc1ca21732e github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index 169da68860..d5b888cc6b 100644 --- a/go.sum +++ b/go.sum @@ -113,7 +113,6 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -176,14 +175,12 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= @@ -210,10 +207,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230707183936-226d2080393a h1:jDoCCEUPnAyPOXO76V4lS1H92gfOO1orMy805gf25bg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230707183936-226d2080393a/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230713175830-cf82e01c0017 h1:5ArSOagq7kDqw+mzVcCB55LDZRgWfH0R5b874GoKYvo= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230713175830-cf82e01c0017/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230719122259-2bc1ca21732e h1:wjDMBLCEkxOPDvIE4LmOHdI/ofJxjcCg0MVSvd/RrXA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230719122259-2bc1ca21732e/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= @@ -246,7 +241,6 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA= diff --git a/internal/caching/cache_space_rooms.go b/internal/caching/cache_space_rooms.go index 2f6d9b0266..90eeb7861c 100644 --- a/internal/caching/cache_space_rooms.go +++ b/internal/caching/cache_space_rooms.go @@ -4,14 +4,14 @@ import "github.com/matrix-org/gomatrixserverlib/fclient" // RoomHierarchy cache caches responses to federated room hierarchy requests (A.K.A. 'space summaries') type RoomHierarchyCache interface { - GetRoomHierarchy(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) - StoreRoomHierarchy(roomID string, r fclient.MSC2946SpacesResponse) + GetRoomHierarchy(roomID string) (r fclient.RoomHierarchyResponse, ok bool) + StoreRoomHierarchy(roomID string, r fclient.RoomHierarchyResponse) } -func (c Caches) GetRoomHierarchy(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) { +func (c Caches) GetRoomHierarchy(roomID string) (r fclient.RoomHierarchyResponse, ok bool) { return c.RoomHierarchies.Get(roomID) } -func (c Caches) StoreRoomHierarchy(roomID string, r fclient.MSC2946SpacesResponse) { +func (c Caches) StoreRoomHierarchy(roomID string, r fclient.RoomHierarchyResponse) { c.RoomHierarchies.Set(roomID, r) } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index b5d45e8e5d..16e547578f 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -35,7 +35,7 @@ type Caches struct { RoomServerEventTypes Cache[types.EventTypeNID, string] // eventType NID -> eventType FederationPDUs Cache[int64, *types.HeaderedEvent] // queue NID -> PDU FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU - RoomHierarchies Cache[string, fclient.MSC2946SpacesResponse] // room ID -> space response + RoomHierarchies Cache[string, fclient.RoomHierarchyResponse] // room ID -> space response LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID } diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index d970cfad4b..97ea9548f6 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -147,7 +147,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm MaxAge: lesserOf(time.Hour/2, maxAge), }, }, - RoomHierarchies: &RistrettoCachePartition[string, fclient.MSC2946SpacesResponse]{ // room ID -> space response + RoomHierarchies: &RistrettoCachePartition[string, fclient.RoomHierarchyResponse]{ // room ID -> space response cache: cache, Prefix: spaceSummaryRoomsCache, Mutable: true, diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 826bb06117..28b381d355 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -132,7 +132,7 @@ type QueryRoomHierarchyAPI interface { // // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it // can be cached. - QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.MSC2946Room, *RoomHierarchyWalker, error) + QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error) } // API functions required by the syncapi diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index b05db74291..7274be520c 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -38,12 +38,12 @@ import ( // // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it // can be cached. -func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.MSC2946Room, *roomserver.RoomHierarchyWalker, error) { +func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *roomserver.RoomHierarchyWalker, error) { if authorised, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised { return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} } - discoveredRooms := []fclient.MSC2946Room{} + discoveredRooms := []fclient.RoomHierarchyRoom{} // Copy unvisited and processed to avoid modifying original walker (which is typically in cache) unvisited := make([]roomserver.RoomHierarchyWalkerQueuedRoom, len(walker.Unvisited)) @@ -81,7 +81,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r } // Collect rooms/events to send back (either locally or fetched via federation) - var discoveredChildEvents []fclient.MSC2946StrippedEvent + var discoveredChildEvents []fclient.RoomHierarchyStrippedEvent // If we know about this room and the caller is authorised (joined/world_readable) then pull // events locally @@ -112,7 +112,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) - discoveredRooms = append(discoveredRooms, fclient.MSC2946Room{ + discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{ PublicRoom: *pubRoom, RoomType: roomType, ChildrenState: events, @@ -377,7 +377,7 @@ func roomExists(ctx context.Context, querier *Queryer, roomID spec.RoomID) bool // federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was // unsuccessful. -func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, suggestedOnly bool, roomID spec.RoomID, vias []string) *fclient.MSC2946SpacesResponse { +func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, suggestedOnly bool, roomID spec.RoomID, vias []string) *fclient.RoomHierarchyResponse { // only do federated requests for client requests if caller.Device() == nil { return nil @@ -401,12 +401,12 @@ func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.Devic } // ensure nil slices are empty as we send this to the client sometimes if res.Room.ChildrenState == nil { - res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} + res.Room.ChildrenState = []fclient.RoomHierarchyStrippedEvent{} } for i := 0; i < len(res.Children); i++ { child := res.Children[i] if child.ChildrenState == nil { - child.ChildrenState = []fclient.MSC2946StrippedEvent{} + child.ChildrenState = []fclient.RoomHierarchyStrippedEvent{} } res.Children[i] = child } @@ -418,7 +418,7 @@ func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.Devic } // references returns all child references pointing to or from this room. -func childReferences(ctx context.Context, querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ([]fclient.MSC2946StrippedEvent, error) { +func childReferences(ctx context.Context, querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ([]fclient.RoomHierarchyStrippedEvent, error) { createTuple := gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomCreate, StateKey: "", @@ -447,12 +447,12 @@ func childReferences(ctx context.Context, querier *Queryer, suggestedOnly bool, } roomType := createContent.RoomType if roomType != spec.MSpace { - return []fclient.MSC2946StrippedEvent{}, nil + return []fclient.RoomHierarchyStrippedEvent{}, nil } } delete(res.StateEvents, createTuple) - el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) + el := make([]fclient.RoomHierarchyStrippedEvent, 0, len(res.StateEvents)) for _, ev := range res.StateEvents { content := gjson.ParseBytes(ev.Content()) // only return events that have a `via` key as per MSC1772 @@ -492,11 +492,11 @@ func publicRoomsChunk(ctx context.Context, querier *Queryer, roomID spec.RoomID) return &pubRooms[0] } -func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { +func stripped(ev gomatrixserverlib.PDU) *fclient.RoomHierarchyStrippedEvent { if ev.StateKey() == nil { return nil } - return &fclient.MSC2946StrippedEvent{ + return &fclient.RoomHierarchyStrippedEvent{ Type: ev.Type(), StateKey: *ev.StateKey(), Content: ev.Content(), From fad46586ba20d372ba681bd006082086e693321b Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Wed, 19 Jul 2023 16:52:37 +0100 Subject: [PATCH 20/21] bump helm chart version (0.13.1) --- helm/dendrite/Chart.yaml | 4 +- helm/dendrite/README.md | 252 +++++++++++++++++++------------------- helm/dendrite/values.yaml | 2 +- 3 files changed, 131 insertions(+), 127 deletions(-) diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 668fd84ec2..8fa06dd970 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.13.0" -appVersion: "0.13.0" +version: "0.13.1" +appVersion: "0.13.1" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 53cb187ab2..7eabe88e6a 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.0](https://img.shields.io/badge/Version-0.13.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.0](https://img.shields.io/badge/AppVersion-0.13.0-informational?style=flat-square) +![Version: 0.13.1](https://img.shields.io/badge/Version-0.13.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.1](https://img.shields.io/badge/AppVersion-0.13.1-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -35,131 +35,133 @@ Create a folder `appservices` and place your configurations in there. The confi * ## Requirements -| Repository | Name | Version | -| ---------------------------------- | ---------- | ------- | -| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | ## Values -| Key | Type | Default | Description | -| ------------------------------------------------------------ | ------ | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| image.repository | string | `"ghcr.io/matrix-org/dendrite-monolith"` | Docker repository/image to use | -| image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | -| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | -| signing_key.create | bool | `true` | Create a new signing key, if not exists | -| signing_key.existingSecret | string | `""` | Use an existing secret | -| resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | -| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | -| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | -| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | -| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | -| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | -| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | -| extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | -| extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | -| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | -| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | -| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | -| dendrite_config.version | int | `2` | | -| dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | -| dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | -| dendrite_config.global.well_known_server_name | string | `""` | The server name to delegate server-server communications to, with optional port e.g. localhost:443 | -| dendrite_config.global.well_known_client_name | string | `""` | The server name to delegate client-server communications to, with optional port e.g. localhost:443 | -| dendrite_config.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Lists of domains that the server will trust as identity servers to verify third party identifiers such as phone numbers and email addresses. | -| dendrite_config.global.old_private_keys | string | `nil` | The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) to old signing keys that were formerly in use on this domain name. These keys will not be used for federation request or event signing, but will be provided to any other homeserver that asks when trying to verify old events. | -| dendrite_config.global.disable_federation | bool | `false` | Disable federation. Dendrite will not be able to make any outbound HTTP requests to other servers and the federation API will not be exposed. | -| dendrite_config.global.key_validity_period | string | `"168h0m0s"` | | -| dendrite_config.global.database.connection_string | string | `""` | The connection string for connections to Postgres. This will be set automatically if using the Postgres dependency | -| dendrite_config.global.database.max_open_conns | int | `90` | Default database maximum open connections | -| dendrite_config.global.database.max_idle_conns | int | `5` | Default database maximum idle connections | -| dendrite_config.global.database.conn_max_lifetime | int | `-1` | Default database maximum lifetime | -| dendrite_config.global.jetstream.storage_path | string | `"/data/jetstream"` | Persistent directory to store JetStream streams in. | -| dendrite_config.global.jetstream.addresses | list | `[]` | NATS JetStream server addresses if not using internal NATS. | -| dendrite_config.global.jetstream.topic_prefix | string | `"Dendrite"` | The prefix for JetStream streams | -| dendrite_config.global.jetstream.in_memory | bool | `false` | Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) | -| dendrite_config.global.jetstream.disable_tls_validation | bool | `true` | Disables TLS validation. This should **NOT** be used in production. | -| dendrite_config.global.cache.max_size_estimated | string | `"1gb"` | The estimated maximum size for the global cache in bytes, or in terabytes, gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or 'kb' suffix is specified. Note that this is not a hard limit, nor is it a memory limit for the entire process. A cache that is too small may ultimately provide little or no benefit. | -| dendrite_config.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory before it will be evicted and/or refreshed from the database. Lower values result in easier admission of new cache entries but may also increase database load in comparison to higher values, so adjust conservatively. Higher values may make it harder for new items to make it into the cache, e.g. if new rooms suddenly become popular. | -| dendrite_config.global.report_stats.enabled | bool | `false` | Configures phone-home statistics reporting. These statistics contain the server name, number of active users and some information on your deployment config. We use this information to understand how Dendrite is being used in the wild. | -| dendrite_config.global.report_stats.endpoint | string | `"https://matrix.org/report-usage-stats/push"` | Endpoint to report statistics to. | -| dendrite_config.global.presence.enable_inbound | bool | `false` | Controls whether we receive presence events from other servers | -| dendrite_config.global.presence.enable_outbound | bool | `false` | Controls whether we send presence events for our local users to other servers. (_May increase CPU/memory usage_) | -| dendrite_config.global.server_notices.enabled | bool | `false` | Server notices allows server admins to send messages to all users on the server. | -| dendrite_config.global.server_notices.local_part | string | `"_server"` | The local part for the user sending server notices. | -| dendrite_config.global.server_notices.display_name | string | `"Server Alerts"` | The display name for the user sending server notices. | -| dendrite_config.global.server_notices.avatar_url | string | `""` | The avatar URL (as a mxc:// URL) name for the user sending server notices. | -| dendrite_config.global.server_notices.room_name | string | `"Server Alerts"` | | -| dendrite_config.global.metrics.enabled | bool | `false` | Whether or not Prometheus metrics are enabled. | -| dendrite_config.global.metrics.basic_auth.user | string | `"metrics"` | HTTP basic authentication username | -| dendrite_config.global.metrics.basic_auth.password | string | `"metrics"` | HTTP basic authentication password | -| dendrite_config.global.dns_cache.enabled | bool | `false` | Whether or not the DNS cache is enabled. | -| dendrite_config.global.dns_cache.cache_size | int | `256` | Maximum number of entries to hold in the DNS cache | -| dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | -| dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | -| dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | -| dendrite_config.mscs | object | `{"mscs":["msc2836"]}` | Configuration for experimental MSC's. (Valid values are: msc2836) | -| dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | -| dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | -| dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | -| dendrite_config.client_api.guests_disabled | bool | `true` | | -| dendrite_config.client_api.registration_shared_secret | string | `""` | If set, allows registration by anyone who knows the shared secret, regardless of whether registration is otherwise disabled. | -| dendrite_config.client_api.enable_registration_captcha | bool | `false` | enable reCAPTCHA registration | -| dendrite_config.client_api.recaptcha_public_key | string | `""` | reCAPTCHA public key | -| dendrite_config.client_api.recaptcha_private_key | string | `""` | reCAPTCHA private key | -| dendrite_config.client_api.recaptcha_bypass_secret | string | `""` | reCAPTCHA bypass secret | -| dendrite_config.client_api.recaptcha_siteverify_api | string | `""` | | -| dendrite_config.client_api.turn.turn_user_lifetime | string | `"24h"` | Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | -| dendrite_config.client_api.turn.turn_uris | list | `[]` | | -| dendrite_config.client_api.turn.turn_shared_secret | string | `""` | | -| dendrite_config.client_api.turn.turn_username | string | `""` | The TURN username | -| dendrite_config.client_api.turn.turn_password | string | `""` | The TURN password | -| dendrite_config.client_api.rate_limiting.enabled | bool | `true` | Enable rate limiting | -| dendrite_config.client_api.rate_limiting.threshold | int | `20` | After how many requests a rate limit should be activated | -| dendrite_config.client_api.rate_limiting.cooloff_ms | int | `500` | Cooloff time in milliseconds | -| dendrite_config.client_api.rate_limiting.exempt_user_ids | string | `nil` | Users which should be exempt from rate limiting | -| dendrite_config.federation_api.send_max_retries | int | `16` | Federation failure threshold. How many consecutive failures that we should tolerate when sending federation requests to a specific server. The backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. The default value is 16 if not specified, which is circa 18 hours. | -| dendrite_config.federation_api.disable_tls_validation | bool | `false` | Disable TLS validation. This should **NOT** be used in production. | -| dendrite_config.federation_api.prefer_direct_fetch | bool | `false` | | -| dendrite_config.federation_api.disable_http_keepalives | bool | `false` | Prevents Dendrite from keeping HTTP connections open for reuse for future requests. Connections will be closed quicker but we may spend more time on TLS handshakes instead. | -| dendrite_config.federation_api.key_perspectives | list | See value.yaml | Perspective keyservers, to use as a backup when direct key fetch requests don't succeed. | -| dendrite_config.media_api.base_path | string | `"/data/media_store"` | The path to store media files (e.g. avatars) in | -| dendrite_config.media_api.max_file_size_bytes | int | `10485760` | The max file size for uploaded media files | -| dendrite_config.media_api.dynamic_thumbnails | bool | `false` | | -| dendrite_config.media_api.max_thumbnail_generators | int | `10` | The maximum number of simultaneous thumbnail generators to run. | -| dendrite_config.media_api.thumbnail_sizes | list | See value.yaml | A list of thumbnail sizes to be generated for media content. | -| dendrite_config.sync_api.real_ip_header | string | `"X-Real-IP"` | This option controls which HTTP header to inspect to find the real remote IP address of the client. This is likely required if Dendrite is running behind a reverse proxy server. | -| dendrite_config.sync_api.search | object | `{"enabled":true,"index_path":"/data/search","language":"en"}` | Configuration for the full-text search engine. | -| dendrite_config.sync_api.search.enabled | bool | `true` | Whether fulltext search is enabled. | -| dendrite_config.sync_api.search.index_path | string | `"/data/search"` | The path to store the search index in. | -| dendrite_config.sync_api.search.language | string | `"en"` | The language most likely to be used on the server - used when indexing, to ensure the returned results match expectations. A full list of possible languages can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) | -| dendrite_config.user_api.bcrypt_cost | int | `10` | bcrypt cost to use when hashing passwords. (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) | -| dendrite_config.user_api.openid_token_lifetime_ms | int | `3600000` | OpenID Token lifetime in milliseconds. | -| dendrite_config.user_api.push_gateway_disable_tls_validation | bool | `false` | | -| dendrite_config.user_api.auto_join_rooms | list | `[]` | Rooms to join users to after registration | -| dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | -| postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | -| postgresql.image.repository | string | `"bitnami/postgresql"` | | -| postgresql.image.tag | string | `"15.1.0"` | | -| postgresql.auth.username | string | `"dendrite"` | | -| postgresql.auth.password | string | `"changeme"` | | -| postgresql.auth.database | string | `"dendrite"` | | -| postgresql.persistence.enabled | bool | `false` | | -| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | -| ingress.hosts | list | `[]` | | -| ingress.className | string | `""` | | -| ingress.hostName | string | `""` | | -| ingress.annotations | object | `{}` | Extra, custom annotations | -| ingress.tls | list | `[]` | | -| service.type | string | `"ClusterIP"` | | -| service.port | int | `8008` | | -| prometheus.servicemonitor.enabled | bool | `false` | Enable ServiceMonitor for Prometheus-Operator for scrape metric-endpoint | -| prometheus.servicemonitor.labels | object | `{}` | Extra Labels on ServiceMonitor for selector of Prometheus Instance | -| prometheus.rules.enabled | bool | `false` | Enable PrometheusRules for Prometheus-Operator for setup alerting | -| prometheus.rules.labels | object | `{}` | Extra Labels on PrometheusRules for selector of Prometheus Instance | -| prometheus.rules.additionalRules | list | `[]` | additional alertrules (no default alertrules are provided) | -| grafana.dashboards.enabled | bool | `false` | | -| grafana.dashboards.labels | object | `{"grafana_dashboard":"1"}` | Extra Labels on ConfigMap for selector of grafana sidecar | -| grafana.dashboards.annotations | object | `{}` | Extra Annotations on ConfigMap additional config in grafana sidecar | +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| image.repository | string | `"ghcr.io/matrix-org/dendrite-monolith"` | Docker repository/image to use | +| image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| signing_key.create | bool | `true` | Create a new signing key, if not exists | +| signing_key.existingSecret | string | `""` | Use an existing secret | +| resources | object | sets some sane default values | Default resource requests/limits. | +| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. | +| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | +| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | +| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | +| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | +| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | +| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | +| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | +| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | +| extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | +| extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | +| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | +| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | +| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | +| dendrite_config.version | int | `2` | | +| dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | +| dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | +| dendrite_config.global.well_known_server_name | string | `""` | The server name to delegate server-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.well_known_client_name | string | `""` | The server name to delegate client-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Lists of domains that the server will trust as identity servers to verify third party identifiers such as phone numbers and email addresses. | +| dendrite_config.global.old_private_keys | string | `nil` | The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) to old signing keys that were formerly in use on this domain name. These keys will not be used for federation request or event signing, but will be provided to any other homeserver that asks when trying to verify old events. | +| dendrite_config.global.disable_federation | bool | `false` | Disable federation. Dendrite will not be able to make any outbound HTTP requests to other servers and the federation API will not be exposed. | +| dendrite_config.global.key_validity_period | string | `"168h0m0s"` | | +| dendrite_config.global.database.connection_string | string | `""` | The connection string for connections to Postgres. This will be set automatically if using the Postgres dependency | +| dendrite_config.global.database.max_open_conns | int | `90` | Default database maximum open connections | +| dendrite_config.global.database.max_idle_conns | int | `5` | Default database maximum idle connections | +| dendrite_config.global.database.conn_max_lifetime | int | `-1` | Default database maximum lifetime | +| dendrite_config.global.jetstream.storage_path | string | `"/data/jetstream"` | Persistent directory to store JetStream streams in. | +| dendrite_config.global.jetstream.addresses | list | `[]` | NATS JetStream server addresses if not using internal NATS. | +| dendrite_config.global.jetstream.topic_prefix | string | `"Dendrite"` | The prefix for JetStream streams | +| dendrite_config.global.jetstream.in_memory | bool | `false` | Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) | +| dendrite_config.global.jetstream.disable_tls_validation | bool | `true` | Disables TLS validation. This should **NOT** be used in production. | +| dendrite_config.global.cache.max_size_estimated | string | `"1gb"` | The estimated maximum size for the global cache in bytes, or in terabytes, gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or 'kb' suffix is specified. Note that this is not a hard limit, nor is it a memory limit for the entire process. A cache that is too small may ultimately provide little or no benefit. | +| dendrite_config.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory before it will be evicted and/or refreshed from the database. Lower values result in easier admission of new cache entries but may also increase database load in comparison to higher values, so adjust conservatively. Higher values may make it harder for new items to make it into the cache, e.g. if new rooms suddenly become popular. | +| dendrite_config.global.report_stats.enabled | bool | `false` | Configures phone-home statistics reporting. These statistics contain the server name, number of active users and some information on your deployment config. We use this information to understand how Dendrite is being used in the wild. | +| dendrite_config.global.report_stats.endpoint | string | `"https://matrix.org/report-usage-stats/push"` | Endpoint to report statistics to. | +| dendrite_config.global.presence.enable_inbound | bool | `false` | Controls whether we receive presence events from other servers | +| dendrite_config.global.presence.enable_outbound | bool | `false` | Controls whether we send presence events for our local users to other servers. (_May increase CPU/memory usage_) | +| dendrite_config.global.server_notices.enabled | bool | `false` | Server notices allows server admins to send messages to all users on the server. | +| dendrite_config.global.server_notices.local_part | string | `"_server"` | The local part for the user sending server notices. | +| dendrite_config.global.server_notices.display_name | string | `"Server Alerts"` | The display name for the user sending server notices. | +| dendrite_config.global.server_notices.avatar_url | string | `""` | The avatar URL (as a mxc:// URL) name for the user sending server notices. | +| dendrite_config.global.server_notices.room_name | string | `"Server Alerts"` | | +| dendrite_config.global.metrics.enabled | bool | `false` | Whether or not Prometheus metrics are enabled. | +| dendrite_config.global.metrics.basic_auth.user | string | `"metrics"` | HTTP basic authentication username | +| dendrite_config.global.metrics.basic_auth.password | string | `"metrics"` | HTTP basic authentication password | +| dendrite_config.global.dns_cache.enabled | bool | `false` | Whether or not the DNS cache is enabled. | +| dendrite_config.global.dns_cache.cache_size | int | `256` | Maximum number of entries to hold in the DNS cache | +| dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | +| dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | +| dendrite_config.mscs | object | `{"mscs":[]}` | Configuration for experimental MSC's. (Valid values are: msc2836) | +| dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | +| dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | +| dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | +| dendrite_config.client_api.guests_disabled | bool | `true` | | +| dendrite_config.client_api.registration_shared_secret | string | `""` | If set, allows registration by anyone who knows the shared secret, regardless of whether registration is otherwise disabled. | +| dendrite_config.client_api.enable_registration_captcha | bool | `false` | enable reCAPTCHA registration | +| dendrite_config.client_api.recaptcha_public_key | string | `""` | reCAPTCHA public key | +| dendrite_config.client_api.recaptcha_private_key | string | `""` | reCAPTCHA private key | +| dendrite_config.client_api.recaptcha_bypass_secret | string | `""` | reCAPTCHA bypass secret | +| dendrite_config.client_api.recaptcha_siteverify_api | string | `""` | | +| dendrite_config.client_api.turn.turn_user_lifetime | string | `"24h"` | Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.client_api.turn.turn_uris | list | `[]` | | +| dendrite_config.client_api.turn.turn_shared_secret | string | `""` | | +| dendrite_config.client_api.turn.turn_username | string | `""` | The TURN username | +| dendrite_config.client_api.turn.turn_password | string | `""` | The TURN password | +| dendrite_config.client_api.rate_limiting.enabled | bool | `true` | Enable rate limiting | +| dendrite_config.client_api.rate_limiting.threshold | int | `20` | After how many requests a rate limit should be activated | +| dendrite_config.client_api.rate_limiting.cooloff_ms | int | `500` | Cooloff time in milliseconds | +| dendrite_config.client_api.rate_limiting.exempt_user_ids | string | `nil` | Users which should be exempt from rate limiting | +| dendrite_config.federation_api.send_max_retries | int | `16` | Federation failure threshold. How many consecutive failures that we should tolerate when sending federation requests to a specific server. The backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. The default value is 16 if not specified, which is circa 18 hours. | +| dendrite_config.federation_api.disable_tls_validation | bool | `false` | Disable TLS validation. This should **NOT** be used in production. | +| dendrite_config.federation_api.prefer_direct_fetch | bool | `false` | | +| dendrite_config.federation_api.disable_http_keepalives | bool | `false` | Prevents Dendrite from keeping HTTP connections open for reuse for future requests. Connections will be closed quicker but we may spend more time on TLS handshakes instead. | +| dendrite_config.federation_api.key_perspectives | list | See value.yaml | Perspective keyservers, to use as a backup when direct key fetch requests don't succeed. | +| dendrite_config.media_api.base_path | string | `"/data/media_store"` | The path to store media files (e.g. avatars) in | +| dendrite_config.media_api.max_file_size_bytes | int | `10485760` | The max file size for uploaded media files | +| dendrite_config.media_api.dynamic_thumbnails | bool | `false` | | +| dendrite_config.media_api.max_thumbnail_generators | int | `10` | The maximum number of simultaneous thumbnail generators to run. | +| dendrite_config.media_api.thumbnail_sizes | list | See value.yaml | A list of thumbnail sizes to be generated for media content. | +| dendrite_config.sync_api.real_ip_header | string | `"X-Real-IP"` | This option controls which HTTP header to inspect to find the real remote IP address of the client. This is likely required if Dendrite is running behind a reverse proxy server. | +| dendrite_config.sync_api.search | object | `{"enabled":true,"index_path":"/data/search","language":"en"}` | Configuration for the full-text search engine. | +| dendrite_config.sync_api.search.enabled | bool | `true` | Whether fulltext search is enabled. | +| dendrite_config.sync_api.search.index_path | string | `"/data/search"` | The path to store the search index in. | +| dendrite_config.sync_api.search.language | string | `"en"` | The language most likely to be used on the server - used when indexing, to ensure the returned results match expectations. A full list of possible languages can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) | +| dendrite_config.user_api.bcrypt_cost | int | `10` | bcrypt cost to use when hashing passwords. (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) | +| dendrite_config.user_api.openid_token_lifetime_ms | int | `3600000` | OpenID Token lifetime in milliseconds. | +| dendrite_config.user_api.push_gateway_disable_tls_validation | bool | `false` | | +| dendrite_config.user_api.auto_join_rooms | list | `[]` | Rooms to join users to after registration | +| dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | +| postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | +| postgresql.image.repository | string | `"bitnami/postgresql"` | | +| postgresql.image.tag | string | `"15.1.0"` | | +| postgresql.auth.username | string | `"dendrite"` | | +| postgresql.auth.password | string | `"changeme"` | | +| postgresql.auth.database | string | `"dendrite"` | | +| postgresql.persistence.enabled | bool | `false` | | +| ingress.enabled | bool | `false` | Create an ingress for the deployment | +| ingress.className | string | `""` | The ingressClass to use. Will be converted to annotation if not yet supported. | +| ingress.annotations | object | `{}` | Extra, custom annotations | +| ingress.hostName | string | `""` | The ingress hostname for your matrix server. Should align with the server_name and well_known_* hosts. If not set, generated from the dendrite_config values. | +| ingress.tls | list | `[]` | TLS configuration. Should contain information for the server_name and well-known hosts. Alternatively, set tls.generate=true to generate defaults based on the dendrite_config. | +| service.type | string | `"ClusterIP"` | | +| service.port | int | `8008` | | +| prometheus.servicemonitor.enabled | bool | `false` | Enable ServiceMonitor for Prometheus-Operator for scrape metric-endpoint | +| prometheus.servicemonitor.labels | object | `{}` | Extra Labels on ServiceMonitor for selector of Prometheus Instance | +| prometheus.rules.enabled | bool | `false` | Enable PrometheusRules for Prometheus-Operator for setup alerting | +| prometheus.rules.labels | object | `{}` | Extra Labels on PrometheusRules for selector of Prometheus Instance | +| prometheus.rules.additionalRules | list | `[]` | additional alertrules (no default alertrules are provided) | +| grafana.dashboards.enabled | bool | `false` | | +| grafana.dashboards.labels | object | `{"grafana_dashboard":"1"}` | Extra Labels on ConfigMap for selector of grafana sidecar | +| grafana.dashboards.annotations | object | `{}` | Extra Annotations on ConfigMap additional config in grafana sidecar | ## Monitoring @@ -187,3 +189,5 @@ grafana: ``` PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`) +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) \ No newline at end of file diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index e3ad0f963f..8a72f66930 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -213,7 +213,7 @@ dendrite_config: # -- Configuration for experimental MSC's. (Valid values are: msc2836) mscs: - mscs: + mscs: [] # A list of enabled MSC's # Currently valid values are: # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) From 31f86e69f78d5f7439fad730f0ebb021238ba851 Mon Sep 17 00:00:00 2001 From: Sam Wedgwood Date: Thu, 20 Jul 2023 14:09:55 +0100 Subject: [PATCH 21/21] bump GMSL --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c936b41515..77f514190a 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230719122259-2bc1ca21732e + github.com/matrix-org/gomatrixserverlib v0.0.0-20230720130651-c87b4eaee74b github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 diff --git a/go.sum b/go.sum index d5b888cc6b..39f0a53442 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230719122259-2bc1ca21732e h1:wjDMBLCEkxOPDvIE4LmOHdI/ofJxjcCg0MVSvd/RrXA= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230719122259-2bc1ca21732e/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230720130651-c87b4eaee74b h1:jnrdkecF6zsq02eC/XXo0B+Ohtpx0fH7jVTQQ9EyIqo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230720130651-c87b4eaee74b/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=