Merge branch 'develop' into shay/batch_persist

.ci/scripts/test_export_data_command.sh

@@ -32,7 +32,7 @@ else
 fi
 
 # Create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+psql -c "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"

.ci/scripts/test_synapse_port_db.sh

@@ -2,27 +2,27 @@
 #
 # Test script for 'synapse_port_db'.
 #   - configures synapse and a postgres server.
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
+#   - runs the port script on a prepopulated test sqlite db. Checks that the
+#     return code is zero.
+#   - reruns the port script on the same sqlite db, targetting the same postgres db.
+#     Checks that the return code is zero.
+#   - runs the port script against a new sqlite db. Checks the return code is zero.
 #
 # Expects Synapse to have been already installed with `poetry install --extras postgres`.
 # Expects `poetry` to be available on the `PATH`.
 
-set -xe
+set -xe -o pipefail
 cd "$(dirname "$0")/../.."
 
 echo "--- Generate the signing key"
-
-# Generate the server's signing key.
 poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
+# Make sure the SQLite3 database is using the latest schema and has no pending background updates.
 poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+psql -c "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
@@ -45,9 +45,23 @@ rm .ci/test_db.db
 poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
+psql \
+  -c "DROP DATABASE synapse" \
+  -c "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
 poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+
+echo "--- Create a brand new postgres database from schema"
+cp .ci/postgres-config.yaml .ci/postgres-config-unported.yaml
+sed -i -e 's/database: synapse/database: synapse_unported/' .ci/postgres-config-unported.yaml
+psql -c "CREATE DATABASE synapse_unported"
+poetry run update_synapse_database --database-config .ci/postgres-config-unported.yaml --run-background-updates
+
+echo "+++ Comparing ported schema with unported schema"
+# Ignore the tables that portdb creates. (Should it tidy them up when the porting is completed?)
+psql synapse -c "DROP TABLE port_from_sqlite3;"
+pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse_unported > unported.sql
+pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse          >   ported.sql
+# By default, `diff` returns zero if there are no changes and nonzero otherwise
+diff -u unported.sql ported.sql | tee schema_diff

.dockerignore

@@ -11,5 +11,6 @@
 !build_rust.py
 
 rust/target
+synapse/*.so
 
 **/__pycache__

.github/workflows/tests.yml

@@ -32,9 +32,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: pip install .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          extras: "all"
+      - run: poetry run scripts-dev/generate_sample_config.sh --check
+      - run: poetry run scripts-dev/config-lint.sh
 
   check-schema-delta:
     runs-on: ubuntu-latest
@@ -76,7 +78,6 @@ jobs:
       - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
       - uses: matrix-org/setup-python-poetry@v1
         with:
           extras: "all"
@@ -361,18 +362,22 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
+      - run: sudo apt-get -qq install xmlsec1 postgresql-client
       - uses: matrix-org/setup-python-poetry@v1
         with:
           extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
+        env:
+          PGHOST: localhost
+          PGUSER: postgres
+          PGPASSWORD: postgres
+          PGDATABASE: postgres
+
 
   portdb:
     if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
     strategy:
       matrix:
         include:
@@ -398,12 +403,27 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
+      - run: sudo apt-get -qq install xmlsec1 postgresql-client
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
           extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
+        id: run_tester_script
+        env:
+          PGHOST: localhost
+          PGUSER: postgres
+          PGPASSWORD: postgres
+          PGDATABASE: postgres
+      - name: "Upload schema differences"
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
+        with:
+          name: Schema dumps
+          path: |
+            unported.sql
+            ported.sql
+            schema_diff
 
   complement:
     if: "${{ !failure() && !cancelled() }}"

changelog.d/13162.misc

@@ -0,0 +1 @@
+Bump the minimum dependency of `matrix_common` to 1.3.0 to make use of the `MXCUri` class. Use `MXCUri` to simplify media retention test code.

changelog.d/13589.feature

@@ -0,0 +1 @@
+Keep track when we fail to process a pulled event over federation so we can intelligently back-off in the future.

changelog.d/13723.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where previously rejected events could end up in room state because they pass auth checks given the current state of the room.

changelog.d/13736.feature

@@ -0,0 +1 @@
+Improve validation of request bodies for the following client-server API endpoints: [`/account/3pid/add`](https://spec.matrix.org/v1.3/client-server-api/#post_matrixclientv3account3pidadd), [`/account/3pid/bind`](https://spec.matrix.org/v1.3/client-server-api/#post_matrixclientv3account3pidbind), [`/account/3pid/delete`](https://spec.matrix.org/v1.3/client-server-api/#post_matrixclientv3account3piddelete) and [`/account/3pid/unbind`](https://spec.matrix.org/v1.3/client-server-api/#post_matrixclientv3account3pidunbind).

changelog.d/13753.misc

@@ -0,0 +1 @@
+Prepatory work for storing thread IDs for notifications and receipts.

changelog.d/13780.misc

@@ -0,0 +1 @@
+Deduplicate `is_server_notices_room`.

changelog.d/13785.doc

@@ -0,0 +1 @@
+Add docs for common fix of deleting the `matrix_synapse.egg-info/` directory for fixing Python dependency problems.

changelog.d/13788.misc

@@ -0,0 +1 @@
+Remove an old, incorrect migration file.

changelog.d/13794.doc

@@ -0,0 +1 @@
+Update request log format documentation to mention the format used when the authenticated user is controlling another user.

changelog.d/13795.misc

@@ -0,0 +1 @@
+Remove unused method in `synapse.api.auth.Auth`.

changelog.d/13798.misc

@@ -0,0 +1 @@
+Fix a memory leak when running the unit tests.

changelog.d/13801.feature

@@ -0,0 +1 @@
+Add `listeners[x].request_id_header` config to specify which request header to extract and use as the request ID in order to correlate requests from a reverse-proxy.

changelog.d/13802.misc

@@ -0,0 +1 @@
+Use partial indices on SQLite.

changelog.d/13808.misc

@@ -0,0 +1 @@
+Check that portdb generates the same postgres schema as that in the source tree.

changelog.d/13810.feature

@@ -0,0 +1 @@
+Add an admin API endpoint to find a user based on its external ID in an auth provider.

changelog.d/13811.misc

@@ -0,0 +1 @@
+Fix Docker build when Rust .so has been build locally first.

changelog.d/13814.feature

@@ -0,0 +1 @@
+Keep track when we fail to process a pulled event over federation so we can intelligently back-off in the future.

changelog.d/13819.misc

@@ -0,0 +1 @@
+complement: init postgres DB directly inside the target image instead of the base postgres image to fix building using Buildah.

changelog.d/13822.misc

@@ -0,0 +1 @@
+Support providing an index predicate clause when doing upserts.

changelog.d/13825.bugfix

@@ -0,0 +1 @@
+Delete associated data from `event_failed_pull_attempts`, `insertion_events`, `insertion_event_extremities`, `insertion_event_extremities`, `insertion_event_extremities` when purging the room.

changelog.d/13826.bugfix

@@ -0,0 +1 @@
+Fix a long standing bug where device lists would remain cached when remote users left and rejoined the last room shared with the local homeserver.

changelog.d/13827.misc

@@ -0,0 +1 @@
+Minor speedups to linting in CI.

docker/Dockerfile

@@ -31,7 +31,9 @@ ARG PYTHON_VERSION=3.9
 ###
 ### Stage 0: generate requirements.txt
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
+# We hardcode the use of Debian bullseye here because this could change upstream
+# and other Dockerfiles used for testing are expecting bullseye.
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -76,7 +78,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder
 
 # install the OS build deps
 RUN \
@@ -137,7 +139,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ### Stage 2: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-slim
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'

docker/complement/Dockerfile

@@ -17,26 +17,24 @@ ARG SYNAPSE_VERSION=latest
 # the same debian version as Synapse's docker image (so the versions of the
 # shared libraries match).
 
-FROM postgres:13-bullseye AS postgres_base
-    # initialise the database cluster in /var/lib/postgresql
-    RUN gosu postgres initdb --locale=C --encoding=UTF-8 --auth-host password
-
-    # Configure a password and create a database for Synapse
-    RUN echo "ALTER USER postgres PASSWORD 'somesecret'" | gosu postgres postgres --single
-    RUN echo "CREATE DATABASE synapse" | gosu postgres postgres --single
-
 # now build the final image, based on the Synapse image.
 
 FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
     # copy the postgres installation over from the image we built above
     RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=postgres_base /var/lib/postgresql /var/lib/postgresql
-    COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql
+    COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
     RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
     ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
     ENV PGDATA=/var/lib/postgresql/data
 
+    # initialise the database cluster in /var/lib/postgresql
+    RUN gosu postgres initdb --locale=C --encoding=UTF-8 --auth-host password
+
+    # Configure a password and create a database for Synapse
+    RUN echo "ALTER USER postgres PASSWORD 'somesecret'" | gosu postgres postgres --single
+    RUN echo "CREATE DATABASE synapse" | gosu postgres postgres --single
+
     # Extend the shared homeserver config to disable rate-limiting,
     # set Complement's static shared secret, enable registration, amongst other
     # tweaks to get Synapse ready for testing.

docs/admin_api/user_admin_api.md

@@ -1155,3 +1155,41 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+
+### Find a user based on their ID in an auth provider
+
+The API is:
+
+```
+GET /_synapse/admin/v1/auth_providers/$provider/users/$external_id
+```
+
+When a user matched the given ID for the given provider, an HTTP code `200` with a response body like the following is returned:
+
+```json
+{
+    "user_id": "@hello:example.org"
+}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `provider` - The ID of the authentication provider, as advertised by the [`GET /_matrix/client/v3/login`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3login) API in the `m.login.sso` authentication method.
+- `external_id` - The user ID from the authentication provider. Usually corresponds to the `sub` claim for OIDC providers, or to the `uid` attestation for SAML2 providers.
+
+The `external_id` may have characters that are not URL-safe (typically `/`, `:` or `@`), so it is advised to URL-encode those parameters.
+
+**Errors**
+
+Returns a `404` HTTP status code if no user was found, with a response body like this:
+
+```json
+{
+    "errcode":"M_NOT_FOUND",
+    "error":"User not found"
+}
+```
+
+_Added in Synapse 1.68.0._

docs/development/dependencies.md

@@ -126,6 +126,23 @@ context of poetry's venv, without having to run `poetry shell` beforehand.
 poetry install --extras all --remove-untracked
 ```
 
+## ...delete everything and start over from scratch?
+
+```shell
+# Stop the current virtualenv if active
+$ deactivate
+
+# Remove all of the files from the current environment.
+# Don't worry, even though it says "all", this will only
+# remove the Poetry virtualenvs for the current project.
+$ poetry env remove --all
+
+# Reactivate Poetry shell to create the virtualenv again
+$ poetry shell
+# Install everything again
+$ poetry install --extras all
+```
+
 ## ...run a command in the `poetry` virtualenv?
 
 Use `poetry run cmd args` when you need the python virtualenv context.
@@ -256,6 +273,16 @@ from PyPI. (This is what makes poetry seem slow when doing the first
 `poetry install`.) Try `poetry cache list` and `poetry cache clear --all
 <name of cache>` to see if that fixes things.
 
+## Remove outdated egg-info
+
+Delete the `matrix_synapse.egg-info/` directory from the root of your Synapse
+install.
+
+This stores some cached information about dependencies and often conflicts with
+letting Poetry do the right thing.
+
+
+
 ## Try `--verbose` or `--dry-run` arguments.
 
 Sometimes useful to see what poetry's internal logic is.

docs/reverse_proxy.md

@@ -45,6 +45,10 @@ listens to traffic on localhost. (Do not change `bind_addresses` to `127.0.0.1`
 when using a containerized Synapse, as that will prevent it from responding
 to proxied traffic.)
 
+Optionally, you can also set
+[`request_id_header`](../usage/configuration/config_documentation.md#listeners)
+so that the server extracts and re-uses the same request ID format that the
+reverse proxy is using.
 
 ## Reverse-proxy configuration examples