Add endpoints for backfilling history (MSC2716)

changelog.d/9247.feature

@@ -0,0 +1 @@
+Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)).

scripts-dev/complement.sh

@@ -46,4 +46,4 @@ if [[ -n "$1" ]]; then
 fi
 
 # Run the tests!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -tags msc2716 -v -count=1 ./tests/main_test.go ./tests/msc2716_test.go 

synapse/api/auth.py

@@ -87,11 +87,13 @@ def __init__(self, hs):
     async def check_from_context(
         self, room_version: str, event, context, do_sig_check=True
     ):
-        prev_state_ids = await context.get_prev_state_ids()
-        auth_events_ids = self.compute_auth_events(
-            event, prev_state_ids, for_verification=True
-        )
-        auth_events = await self.store.get_events(auth_events_ids)
+        auth_event_ids = event.auth_event_ids()
+        if auth_event_ids is None:
+            prev_state_ids = await context.get_prev_state_ids()
+            auth_event_ids = self.compute_auth_events(
+                event, prev_state_ids, for_verification=True
+            )
+        auth_events = await self.store.get_events(auth_event_ids)
         auth_events = {(e.type, e.state_key): e for e in auth_events.values()}
 
         room_version_obj = KNOWN_ROOM_VERSIONS[room_version]

synapse/api/constants.py

@@ -106,6 +106,9 @@ class EventTypes:
     MSC1772_SPACE_CHILD = "org.matrix.msc1772.space.child"
     MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent"
 
+    MSC2716_INSERTION = "org.matrix.msc2716.insertion"
+    MSC2716_MARKER = "org.matrix.msc2716.marker"
+
 
 class EduTypes:
     Presence = "m.presence"
@@ -169,6 +172,13 @@ class EventContentFields:
     # cf https://github.com/matrix-org/matrix-doc/pull/1772
     MSC1772_ROOM_TYPE = "org.matrix.msc1772.type"
 
+    # For "insertion" events
+    MSC2716_NEXT_CHUNK_ID = "org.matrix.msc2716.next_chunk_id"
+    # Used on normal message events to indicate where the chunk connects to
+    MSC2716_CHUNK_ID = "org.matrix.msc2716.chunk_id"
+    # For "marker" events
+    MSC2716_PREV_INSERTION = "org.matrix.msc2716.prev_insertion"
+
 
 class RoomEncryptionAlgorithms:
     MEGOLM_V1_AES_SHA2 = "m.megolm.v1.aes-sha2"

synapse/config/experimental.py

@@ -36,3 +36,5 @@ def read_config(self, config: JsonDict, **kwargs):
 
         # MSC3026 (busy presence state)
         self.msc3026_enabled = experimental.get("msc3026_enabled", False)  # type: bool
+        # MSC2716 (backfill existing history)
+        self.msc2716_enabled = experimental.get("msc2716_enabled", False)  # type: bool

synapse/events/builder.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import attr
@@ -34,6 +35,8 @@
 from synapse.util import Clock
 from synapse.util.stringutils import random_string
 
+logger = logging.getLogger(__name__)
+
 
 @attr.s(slots=True, cmp=False, frozen=True)
 class EventBuilder:
@@ -101,6 +104,7 @@ async def build(
         self,
         prev_event_ids: List[str],
         auth_event_ids: Optional[List[str]],
+        inherit_depth: bool = False,
     ) -> EventBase:
         """Transform into a fully signed and hashed event
 
@@ -109,6 +113,8 @@ async def build(
             auth_event_ids: The event IDs to use as the auth events.
                 Should normally be set to None, which will cause them to be calculated
                 based on the room state at the prev_events.
+            inherit_depth: True to inherit the depth from the successor of the most
+                recent event from prev_event_ids. False to progress the depth as normal.
 
         Returns:
             The signed and hashed event.
@@ -132,8 +138,46 @@ async def build(
             auth_events = auth_event_ids
             prev_events = prev_event_ids
 
-        old_depth = await self._store.get_max_depth_of(prev_event_ids)
-        depth = old_depth + 1
+        (
+            most_recent_prev_event_id,
+            most_recent_prev_event_depth,
+        ) = await self._store.get_max_depth_of(prev_event_ids)
+
+        # We want to insert the historical event after the `prev_event` but before the successor event
+        #
+        # We inherit depth from the successor event instead of the `prev_event`
+        # because events returned from `/messages` are first sorted by `topological_ordering`
+        # which is just the `depth` and then tie-break with `stream_ordering`.
+        #
+        # We mark these inserted historical events as "backfilled" which gives them a
+        # negative `stream_ordering`. If we use the same depth as the `prev_event`,
+        # then our historical event will tie-break and be sorted before the `prev_event`
+        # when it should come after.
+        #
+        # We want to use the successor event depth so they appear after `prev_event` because
+        # it has a larger `depth` but before the successor event because the `stream_ordering`
+        # is negative before the successor event.
+        if inherit_depth:
+            successor_event_ids = await self._store.get_successor_events(
+                [most_recent_prev_event_id]
+            )
+
+            # If we can't find any successor events, then it's a forward extremity of
+            # historical messages and we can just inherit from the previous historical
+            # event which we can already assume has the correct depth where we want
+            # to insert into.
+            if not successor_event_ids:
+                depth = most_recent_prev_event_depth
+            else:
+                (
+                    _,
+                    oldest_successor_depth,
+                ) = await self._store.get_min_depth_of(successor_event_ids)
+
+                depth = oldest_successor_depth
+        # Otherwise, progress the depth as normal
+        else:
+            depth = most_recent_prev_event_depth + 1
 
         # we cap depth of generated events, to ensure that they are not
         # rejected by other servers (and so that they can be persisted in

synapse/events/utils.py

@@ -253,13 +253,13 @@ def format_event_for_client_v1(d):
 
 def format_event_for_client_v2(d):
     drop_keys = (
-        "auth_events",
-        "prev_events",
-        "hashes",
-        "signatures",
-        "depth",
-        "origin",
-        "prev_state",
+        # "auth_events",
+        # "prev_events",
+        # "hashes",
+        # "signatures",
+        # "depth",
+        # "origin",
+        # "prev_state",
     )
     for key in drop_keys:
         d.pop(key, None)

synapse/handlers/message.py

@@ -456,6 +456,8 @@ async def create_event(
         prev_event_ids: Optional[List[str]] = None,
         auth_event_ids: Optional[List[str]] = None,
         require_consent: bool = True,
+        outlier: bool = False,
+        inherit_depth: bool = False,
     ) -> Tuple[EventBase, EventContext]:
         """
         Given a dict from a client, create a new event.
@@ -482,6 +484,13 @@ async def create_event(
 
             require_consent: Whether to check if the requester has
                 consented to the privacy policy.
+
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+
+            inherit_depth: True to inherit the depth from the successor of the most
+                recent event from prev_event_ids. False to progress the depth as normal.
         Raises:
             ResourceLimitError if server is blocked to some resource being
             exceeded
@@ -537,11 +546,14 @@ async def create_event(
         if txn_id is not None:
             builder.internal_metadata.txn_id = txn_id
 
+        builder.internal_metadata.outlier = outlier
+
         event, context = await self.create_new_client_event(
             builder=builder,
             requester=requester,
             prev_event_ids=prev_event_ids,
             auth_event_ids=auth_event_ids,
+            inherit_depth=inherit_depth,
         )
 
         # In an ideal world we wouldn't need the second part of this condition. However,
@@ -698,9 +710,12 @@ async def create_and_send_nonmember_event(
         self,
         requester: Requester,
         event_dict: dict,
+        auth_event_ids: Optional[List[str]] = None,
         ratelimit: bool = True,
         txn_id: Optional[str] = None,
         ignore_shadow_ban: bool = False,
+        outlier: bool = False,
+        inherit_depth: bool = False,
     ) -> Tuple[EventBase, int]:
         """
         Creates an event, then sends it.
@@ -710,10 +725,19 @@ async def create_and_send_nonmember_event(
         Args:
             requester: The requester sending the event.
             event_dict: An entire event.
+            auth_event_ids:
+                The event ids to use as the auth_events for the new event.
+                Should normally be left as None, which will cause them to be calculated
+                based on the room state at the prev_events.
             ratelimit: Whether to rate limit this send.
             txn_id: The transaction ID.
             ignore_shadow_ban: True if shadow-banned users should be allowed to
                 send this event.
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+            inherit_depth: True to inherit the depth from the successor of the most
+                recent event from prev_event_ids. False to progress the depth as normal.
 
         Returns:
             The event, and its stream ordering (if deduplication happened,
@@ -752,8 +776,16 @@ async def create_and_send_nonmember_event(
                     assert event.internal_metadata.stream_ordering
                     return event, event.internal_metadata.stream_ordering
 
+            prev_events = event_dict.get("prev_events")
+
             event, context = await self.create_event(
-                requester, event_dict, txn_id=txn_id
+                requester,
+                event_dict,
+                txn_id=txn_id,
+                prev_event_ids=prev_events,
+                auth_event_ids=auth_event_ids,
+                outlier=outlier,
+                inherit_depth=inherit_depth,
             )
 
             assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (
@@ -785,6 +817,7 @@ async def create_new_client_event(
         requester: Optional[Requester] = None,
         prev_event_ids: Optional[List[str]] = None,
         auth_event_ids: Optional[List[str]] = None,
+        inherit_depth: bool = False,
     ) -> Tuple[EventBase, EventContext]:
         """Create a new event for a local client
 
@@ -802,6 +835,9 @@ async def create_new_client_event(
                 Should normally be left as None, which will cause them to be calculated
                 based on the room state at the prev_events.
 
+            inherit_depth: True to inherit the depth from the successor of the most
+                recent event from prev_event_ids. False to progress the depth as normal.
+
         Returns:
             Tuple of created event, context
         """
@@ -825,9 +861,23 @@ async def create_new_client_event(
         ), "Attempting to create an event with no prev_events"
 
         event = await builder.build(
-            prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids
+            prev_event_ids=prev_event_ids,
+            auth_event_ids=auth_event_ids,
+            inherit_depth=inherit_depth,
         )
-        context = await self.state.compute_event_context(event)
+
+        old_state = None
+
+        # Pass on the outlier property from the builder to the event
+        # after it is created
+        if builder.internal_metadata.outlier:
+            event.internal_metadata.outlier = builder.internal_metadata.outlier
+
+            # Calculate the state for outliers that pass in their own `auth_event_ids`
+            if auth_event_ids:
+                old_state = await self.store.get_events_as_list(auth_event_ids)
+
+        context = await self.state.compute_event_context(event, old_state=old_state)
         if requester:
             context.app_service = requester.app_service
 
@@ -1236,13 +1286,21 @@ async def persist_and_notify_client_event(
             if prev_state_ids:
                 raise AuthError(403, "Changing the room create event is forbidden")
 
+        # Mark any `m.historical` messages as backfilled so they don't appear
+        # in `/sync` and have the proper decrementing `stream_ordering` as we import
+        backfilled = False
+        if event.content.get("m.historical", None):
+            backfilled = True
+
         # Note that this returns the event that was persisted, which may not be
         # the same as we passed in if it was deduplicated due transaction IDs.
         (
             event,
             event_pos,
             max_stream_token,
-        ) = await self.storage.persistence.persist_event(event, context=context)
+        ) = await self.storage.persistence.persist_event(
+            event, context=context, backfilled=backfilled
+        )
 
         if self._ephemeral_events_enabled:
             # If there's an expiry timestamp on the event, schedule its expiry.