Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Track device list updates per room. #12321

Merged
merged 30 commits into from
Apr 4, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
b5a9c6b
Operate on room IDs
erikjohnston Mar 25, 2022
6fa639e
Don't copy the hosts list
erikjohnston Mar 28, 2022
ecf98b9
Write to a new `device_lists_changes_in_room` table.
erikjohnston Mar 28, 2022
c5dd83f
Convert to doing everything in a transaction
erikjohnston Mar 28, 2022
bb44214
Track if we've calculated remote hosts
erikjohnston Mar 28, 2022
98fceb3
We actually don't want stream_id to be unique
erikjohnston Mar 28, 2022
eda0e64
Handle room pokes that haven't been converted to outbound pokes
erikjohnston Mar 29, 2022
c7790ab
Deduplicate outbound pokes
erikjohnston Mar 29, 2022
6e9b31a
Add a config option that allows using new code path
erikjohnston Mar 29, 2022
3b2ab93
Add tests for new code path
erikjohnston Mar 29, 2022
f8af30f
Newsfile
erikjohnston Mar 29, 2022
7266580
Fix tests
erikjohnston Mar 29, 2022
f24b70b
Merge remote-tracking branch 'origin/develop' into erikj/device_list_…
erikjohnston Mar 31, 2022
8bd8ee2
Update synapse/storage/schema/main/delta/69/01device_list_oubound_by_…
erikjohnston Apr 4, 2022
56f0913
Update synapse/storage/schema/main/delta/69/01device_list_oubound_by_…
erikjohnston Apr 4, 2022
90d41a0
Encode opentracing context just once.
erikjohnston Apr 4, 2022
c470a12
Rename var
erikjohnston Apr 4, 2022
d030062
Remove `if not room_ids` check.
erikjohnston Apr 4, 2022
ad5d46b
Add unique index
erikjohnston Apr 4, 2022
d5031b0
Note lack of foreign key constraint
erikjohnston Apr 4, 2022
28dacc8
Add comment about stream_id duplicates
erikjohnston Apr 4, 2022
f48527f
Update synapse_port_db
erikjohnston Apr 4, 2022
bd45f19
Inequality the wrong way round
erikjohnston Apr 4, 2022
dee8f55
Add note about 'num_stream_ids'
erikjohnston Apr 4, 2022
3574541
Merge remote-tracking branch 'origin/develop' into erikj/device_list_…
erikjohnston Apr 4, 2022
cf04f1a
Use different stream IDs for device_list_outbound_pokes
erikjohnston Apr 4, 2022
89e10d7
Correctly order device list stream updates
erikjohnston Apr 4, 2022
e54d2d4
Wake up replication after adding otubound pokes
erikjohnston Apr 4, 2022
7d79dee
Apply suggestions from code review
erikjohnston Apr 4, 2022
b61c5c7
Remove get_users_who_share_room_with_user stub in test
erikjohnston Apr 4, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions synapse/handlers/device.py
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,7 @@ async def notify_device_update(
user_id,
device_ids,
hosts=hosts,
room_ids=room_ids,
)

if not position:
Expand Down
1 change: 1 addition & 0 deletions synapse/replication/slave/storage/devices.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def __init__(
extra_tables=[
("user_signature_stream", "stream_id"),
("device_lists_outbound_pokes", "stream_id"),
("device_lists_changes_in_room", "stream_id"),
],
)
device_list_max = self._device_list_id_gen.get_current_token()
Expand Down
1 change: 1 addition & 0 deletions synapse/storage/databases/main/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ def __init__(
extra_tables=[
("user_signature_stream", "stream_id"),
("device_lists_outbound_pokes", "stream_id"),
("device_lists_changes_in_room", "stream_id"),
],
)

Expand Down
59 changes: 57 additions & 2 deletions synapse/storage/databases/main/devices.py
Original file line number Diff line number Diff line change
Expand Up @@ -1506,7 +1506,11 @@ def _update_remote_device_list_cache_txn(
)

async def add_device_change_to_streams(
self, user_id: str, device_ids: Collection[str], hosts: Collection[str]
self,
user_id: str,
device_ids: Collection[str],
hosts: Collection[str],
room_ids: Collection[str],
) -> Optional[int]:
"""Persist that a user's devices have been updated, and which hosts
(if any) should be poked.
Expand All @@ -1516,6 +1520,7 @@ async def add_device_change_to_streams(
device_ids: The IDs of any changed devices. If empty, this function will
return None.
hosts: The remote destinations that should be notified of the change.
room_ids: The rooms that the user is in

Returns:
The maximum stream ID of device list updates that were added to the database, or
Expand All @@ -1535,10 +1540,26 @@ async def add_device_change_to_streams(
stream_ids,
)

if not hosts:
if not room_ids:
return stream_ids[-1]

context = get_active_span_text_map()
async with self._device_list_id_gen.get_next_mult(
len(room_ids) * len(device_ids)
) as stream_ids:
await self.db_pool.runInteraction(
"_add_device_outbound_room_poke_txn",
self._add_device_outbound_room_poke_txn,
user_id,
device_ids,
room_ids,
stream_ids,
context,
)

if not hosts:
return stream_ids[-1]

async with self._device_list_id_gen.get_next_mult(
len(hosts) * len(device_ids)
) as stream_ids:
Expand Down Expand Up @@ -1636,3 +1657,37 @@ def _add_device_outbound_poke_to_stream_txn(
for device_id in device_ids
],
)

def _add_device_outbound_room_poke_txn(
self,
txn: LoggingTransaction,
user_id: str,
device_ids: Iterable[str],
room_ids: Collection[str],
stream_ids: List[str],
context: Dict[str, str],
) -> None:
next_stream_id = iter(stream_ids)
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved

self.db_pool.simple_insert_many_txn(
txn,
table="device_lists_changes_in_room",
keys=(
"stream_id",
"room_id",
"user_id",
"device_id",
"opentracing_context",
),
values=[
(
next(next_stream_id),
room_id,
user_id,
device_id,
json_encoder.encode(context),
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
)
for room_id in room_ids
for device_id in device_ids
],
)
5 changes: 4 additions & 1 deletion synapse/storage/schema/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

SCHEMA_VERSION = 68 # remember to update the list below when updating
SCHEMA_VERSION = 69 # remember to update the list below when updating
"""Represents the expectations made by the codebase about the database schema

This should be incremented whenever the codebase changes its requirements on the
Expand Down Expand Up @@ -58,6 +58,9 @@
- event_reference_hashes is no longer read.
- `events` has `state_key` and `rejection_reason` columns, which are populated for
new events.

Changes in SCHEMA_VERSION = 69:
- We now write to `device_lists_changes_in_room` table.
"""


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/* Copyright 2022 The Matrix.org Foundation C.I.C
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

CREATE TABLE device_lists_changes_in_room (
stream_id BIGINT NOT NULL,
room_id TEXT NOT NULL,
user_id TEXT NOT NULL,
device_id TEXT NOT NULL,
opentracing_context TEXT
);

CREATE UNIQUE INDEX device_lists_changes_in_stream_id ON device_lists_changes_in_room(stream_id);