diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index dc5bcaed14..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,93 +0,0 @@ -version: 2 -# As dependabot is currently only run on a weekly basis, we raise the -# open-pull-requests-limit to 10 (from the default of 5) to better ensure we -# don't continuously grow a backlog of updates. -updates: - - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem - package-ecosystem: "pip" - directory: "/" - open-pull-requests-limit: 10 - versioning-strategy: "increase-if-necessary" - schedule: - interval: "weekly" - # Group patch updates to packages together into a single PR, as they rarely - # if ever contain breaking changes that need to be reviewed separately. - # - # Less PRs means a streamlined review process. - # - # Python packages follow semantic versioning, and tend to only introduce - # breaking changes in major version bumps. Thus, we'll group minor and patch - # versions together. - groups: - minor-and-patches: - applies-to: version-updates - patterns: - - "*" - update-types: - - "minor" - - "patch" - # Prevent pulling packages that were recently updated to help mitigate - # supply chain attacks. 14 days was taken from the recommendation at - # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns - # where the author noted that 9/10 attacks would have been mitigated by a - # two week cooldown. - # - # The cooldown only applies to general updates; security updates will still - # be pulled in as soon as possible. - cooldown: - default-days: 14 - - - package-ecosystem: "docker" - directory: "/docker" - open-pull-requests-limit: 10 - schedule: - interval: "weekly" - # For container versions, breaking changes are also typically only introduced in major - # package bumps. - groups: - minor-and-patches: - applies-to: version-updates - patterns: - - "*" - update-types: - - "minor" - - "patch" - cooldown: - default-days: 14 - - - package-ecosystem: "github-actions" - directory: "/" - open-pull-requests-limit: 10 - schedule: - interval: "weekly" - # Similarly for GitHub Actions, breaking changes are typically only introduced in major - # package bumps. - groups: - minor-and-patches: - applies-to: version-updates - patterns: - - "*" - update-types: - - "minor" - - "patch" - cooldown: - default-days: 14 - - - package-ecosystem: "cargo" - directory: "/" - open-pull-requests-limit: 10 - versioning-strategy: "lockfile-only" - schedule: - interval: "weekly" - # The Rust ecosystem is special in that breaking changes are often introduced - # in minor version bumps, as packages typically stay pre-1.0 for a long time. - # Thus we specifically keep minor version bumps separate in their own PRs. - groups: - patches: - applies-to: version-updates - patterns: - - "*" - update-types: - - "patch" - cooldown: - default-days: 14 diff --git a/.github/workflows/beeper-ci.yaml b/.github/workflows/beeper-ci.yaml new file mode 100644 index 0000000000..168e49d5ea --- /dev/null +++ b/.github/workflows/beeper-ci.yaml @@ -0,0 +1,141 @@ +name: Beep + +on: + push: + branches: ["beeper", "beeper-*"] + pull_request: + + +jobs: + lint-style: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: pip install poetry + - run: poetry install + - run: poetry run ruff check --output-format=github . + + lint-types: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: pip install poetry + - run: poetry install --extras all + - run: poetry run mypy + + # Tests + + test-trial: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: pip install poetry + - run: poetry install --extras all + - run: poetry run trial -j4 tests + + test-sytest: + runs-on: ubuntu-latest + container: + image: matrixdotorg/sytest-synapse:bookworm + volumes: + - ${{ github.workspace }}:/src + env: + SYTEST_BRANCH: e9982eeedd352daaad1e24a067477070b6b5b0b7 + TOP: ${{ github.workspace }} + POSTGRES: 1 + MULTI_POSTGRES: 1 + WOKRERS: 1 + steps: + - uses: actions/checkout@v6 + - name: Run SyTest + run: /bootstrap.sh synapse + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Upload SyTest logs + uses: actions/upload-artifact@v7 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + test-complement: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/checkout@v6 + with: + repository: matrix-org/complement + path: complement + ref: bc2b638b11f5b0a068e39ae51f7ac2782070d14f + - name: Install complement dependencies + run: |- + sudo apt-get -qq update + sudo apt-get install -qqy libolm3 libolm-dev + go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest + pip install poetry + poetry install + - name: Run Complement + run: ./scripts-dev/complement.sh + env: + COMPLEMENT_DIR: complement + + # Builds + + build-python: + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: 1 + steps: + - uses: actions/checkout@v6 + - uses: docker/setup-qemu-action@v4 + - uses: docker/setup-buildx-action@v4 + - uses: docker/login-action@v4 + with: + registry: ${{ secrets.CI_REGISTRY }} + username: ${{ secrets.CI_REGISTRY_USER }} + password: ${{ secrets.CI_REGISTRY_PASSWORD }} + - run: |- + if [ "${{ github.ref_name }}" = "beeper" ]; then + tag=$(cat pyproject.toml | grep -E "^version =" | sed -E 's/^version = "(.+)"$/\1/') + else + tag="${{ github.head_ref || github.ref_name }}" + fi + + docker buildx build \ + --push \ + --platform linux/amd64 \ + --tag ${{ secrets.CI_REGISTRY }}/synapse:$tag-${{ github.sha }} \ + -f docker/Dockerfile \ + . + + if [ "${{ github.ref_name }}" = "beeper" ]; then + docker pull ${{ secrets.CI_REGISTRY }}/synapse:$tag-${{ github.sha }} + docker tag \ + ${{ secrets.CI_REGISTRY }}/synapse:$tag-${{ github.sha }} \ + ${{ secrets.CI_REGISTRY }}/synapse:latest + docker push ${{ secrets.CI_REGISTRY }}/synapse:latest + fi + + # Ensure the image works properly + docker run \ + --entrypoint '' \ + ${{ secrets.CI_REGISTRY }}/synapse:$tag-${{ github.sha }} \ + python -m synapse.app.homeserver --help + + echo "Pushed image: synapse:$tag-${{ github.sha }}" + if [ "${{ github.ref_name }}" = "beeper" ]; then + echo "Pushed image: synapse:latest" + fi diff --git a/README.md b/README.md new file mode 100644 index 0000000000..1867cbc485 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# Synapse: Beeper Edition + +This is Beeper's custom version of synapse, we rebase roughly 25 commits on top of each upstream release with a few Beeper specific modifications. We also have an actual Synapse fork here: [**beeper/synapse-fork**](https://github.com/beeper/synapse-fork) which is where we make changes we expect to merge into upstream. + +## Branching strategy + +We have a bunch of branches. The main development branch for Beeper is +the `beeper` branch. Push commits here and they get built for our +Docker registry. The GitHub Actions logs include the specific image +tag, which is based on the original upstream version as well as the +commit hash. + +We also have versioned `beeper-x.y.z` branches. These are used to +archive the history of the `beeper` branch as it is rebased onto newer +upstream Synapse versions. When rebasing, we copy the current `beeper` +branch back onto the old `beeper-x.y.z` branch for historical +reference, then create a new `beeper-x.y.z` branch based on the new +upstream version and rebase the `beeper` branch onto it, then +force-push that resulting commit into the original `beeper` branch, +which will become the new development target. + +We also have `upstream-x.y.z` branches that just track the upstream +tags that we use as bases for Beeper changes, see the rebase flow +below. + +## CI setup + +Note that we have a separate `beeper-ci.yml` GitHub Actions workflow. +It runs exclusively on the `beeper*` branches, in place of the other +CI workflows that upstream uses and that we have not removed from our +fork. Don't get confused between the two. + +## Rebase flow + +### Create PR + +Here we're upgrading to `v1.96.1`: + +``` +# Make a new branch from the upstream release, we do this so we can create a PR +# of Beeper -> upstream to run tests/confirm we're happy. +git checkout -f v1.96.1 +git checkout -b upstream-1.96.1 +git push -u beeper upstream-1.96.1 + +# Check out the base branch, pull any changes +git checkout beeper +git pull + +# Now create a new branch to rebase +git checkout -b beeper-1.96.1 +# And do the rebase +git rebase v1.96.1 +# fix any conflicts... + +# Push and make a PR from this branch to the upstream one created above +git push -u beeper beeper-1.96.1 +``` + +### Make release + +Once it's ready we just overwrite the `beeper` branch with the new one: + +``` +git checkout beeper-1.96.1 +git push --force beeper beeper +``` diff --git a/beeper/complete_release.sh b/beeper/complete_release.sh new file mode 100755 index 0000000000..1d612ff350 --- /dev/null +++ b/beeper/complete_release.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -euo pipefail +source $(realpath $(dirname $0))/utils.sh + +BEEPER_REMOTE=$(get_beeper_remote) + +VERSION=${1:-} + +if [ -z "$VERSION" ]; then + echo >&2 "Must specify version!" + exit 1 +fi + +echo "Completing Synapse: Beeper Edition version $VERSION" +echo "WARNING: this script will DELETE the branch called: beeper" +read -p "Press enter to continue" + +UPSTREAM_BRANCH=upstream-$VERSION +BEEPER_BRANCH=beeper-$VERSION + +git checkout $BEEPER_BRANCH +git branch -D beeper +git checkout -b beeper +git push --force $BEEPER_REMOTE beeper + +# Cleanup +git branch -D $BEEPER_BRANCH +git push $BEEPER_REMOTE --delete $BEEPER_BRANCH +git branch -D $UPSTREAM_BRANCH +git push $BEEPER_REMOTE --delete $UPSTREAM_BRANCH diff --git a/beeper/prepare_release.sh b/beeper/prepare_release.sh new file mode 100755 index 0000000000..ebd99537c3 --- /dev/null +++ b/beeper/prepare_release.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail +source $(realpath $(dirname $0))/utils.sh + +BEEPER_REMOTE=$(get_beeper_remote) + +VERSION=${1:-} + +if [ -z "$VERSION" ]; then + echo >&2 "Must specify version!" + exit 1 +fi + +STARTING_BRANCH=$(git branch --show-current) + +echo "Preparing Synapse: Beeper Edition version $VERSION" +echo "WARNING: this script will rebase on top of the CURRENT BRANCH: $STARTING_BRANCH" +read -p "Press enter to continue" + +TAG=v$VERSION +UPSTREAM_BRANCH=upstream-$VERSION +BEEPER_BRANCH=beeper-$VERSION + +# Checkout the tag, create upstream branch, push it +echo "Setup branch $UPSTREAM_BRANCH" +git checkout -f $TAG +git checkout -b $UPSTREAM_BRANCH +git push -u $BEEPER_REMOTE $UPSTREAM_BRANCH + +# Switch back to our starting branch, create new version branch from it +echo "Setup branch $BEEPER_BRANCH" +git checkout $STARTING_BRANCH +git checkout -b $BEEPER_BRANCH + +# And rebase against upstream, applying only our Beeper commits +echo "Initiate rebase..." +git rebase $UPSTREAM_BRANCH || read -p "Rebase was a mess, press enter once you fix it" + +git push -u $BEEPER_REMOTE $BEEPER_BRANCH + +echo "OK we done!" +echo "Go HERE and make the PR: https://github.com/beeper/synapse/compare/upstream-$VERSION...beeper-$VERSION?expand=1" diff --git a/beeper/utils.sh b/beeper/utils.sh new file mode 100644 index 0000000000..ba7573dc16 --- /dev/null +++ b/beeper/utils.sh @@ -0,0 +1,23 @@ +function get_upstream_remote() { + for remote in $(git remote); do + url=$(git remote get-url $remote) + if [ "$url" = "git@github.com:element-hq/synapse.git" ]; then + echo $remote + return 0 + fi + done + echo >&2 "No upstream remote found (looking for URL: git@github.com:element-hq/synapse.git)" + return 1 +} + +function get_beeper_remote() { + for remote in $(git remote); do + url=$(git remote get-url $remote) + if [ "$url" = "git@github.com:beeper/synapse.git" ]; then + echo $remote + return 0 + fi + done + echo >&2 "No upstream remote found (looking for URL: git@github.com:beeper/synapse.git)" + return 1 +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 6070d5c355..0a41ad4c1a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -93,6 +93,9 @@ COPY --from=requirements /synapse/requirements.txt /synapse/ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --prefix="/install" --no-deps -r /synapse/requirements.txt +# Beeper: install http antispam +RUN pip install --prefix="/install" --no-deps --no-warn-script-location synapse-http-antispam + # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ COPY rust /synapse/rust/ diff --git a/pyproject.toml b/pyproject.toml index 89bace10a2..8aec6f9858 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -360,6 +360,7 @@ ignore = [ "B023", "E501", "E731", + "G004", ] select = [ # pycodestyle diff --git a/rust/src/push/base_rules.rs b/rust/src/push/base_rules.rs index 47d5289006..064bde85c4 100644 --- a/rust/src/push/base_rules.rs +++ b/rust/src/push/base_rules.rs @@ -83,13 +83,70 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[ default: true, default_enabled: true, }, + // Disable notifications for auto-accepted room invites + // NOTE: this rule must be a higher prio than .m.rule.invite_for_me because + // that will also match the same events. PushRule { - rule_id: Cow::Borrowed("global/override/.m.rule.suppress_notices"), + rule_id: Cow::Borrowed("global/override/.com.beeper.suppress_auto_invite"), + priority_class: 5, + conditions: Cow::Borrowed(&[ + Condition::Known(KnownCondition::EventMatch(EventMatchCondition { + key: Cow::Borrowed("type"), + pattern: Cow::Borrowed("m.room.member"), + })), + Condition::Known(KnownCondition::EventMatch(EventMatchCondition { + key: Cow::Borrowed("content.membership"), + pattern: Cow::Borrowed("invite"), + })), + Condition::Known(KnownCondition::EventMatchType(EventMatchTypeCondition { + key: Cow::Borrowed("state_key"), + pattern_type: Cow::Borrowed(&EventMatchPatternType::UserId), + })), + Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition { + key: Cow::Borrowed("content.fi\\.mau\\.will_auto_accept"), + value: Cow::Borrowed(&SimpleJsonValue::Bool(true)), + })), + ]), + actions: Cow::Borrowed(&[]), + default: true, + default_enabled: true, + }, + // We don't want to notify on edits. Not only can this be confusing in real + // time (2 notifications, one message) but it's especially confusing + // if a bridge needs to edit a previously backfilled message. + PushRule { + rule_id: Cow::Borrowed("global/override/.com.beeper.suppress_edits"), priority_class: 5, conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch( EventMatchCondition { - key: Cow::Borrowed("content.msgtype"), - pattern: Cow::Borrowed("m.notice"), + key: Cow::Borrowed("content.m\\.relates_to.rel_type"), + pattern: Cow::Borrowed("m.replace"), + }, + ))]), + actions: Cow::Borrowed(&[]), + default: true, + default_enabled: true, + }, + PushRule { + rule_id: Cow::Borrowed("global/override/.com.beeper.suppress_send_message_status"), + priority_class: 5, + conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch( + EventMatchCondition { + key: Cow::Borrowed("type"), + pattern: Cow::Borrowed("com.beeper.message_send_status"), + }, + ))]), + actions: Cow::Borrowed(&[]), + default: true, + default_enabled: true, + }, + PushRule { + rule_id: Cow::Borrowed("global/override/.com.beeper.suppress_power_levels"), + priority_class: 5, + conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch( + EventMatchCondition { + key: Cow::Borrowed("type"), + pattern: Cow::Borrowed("m.room.power_levels"), }, ))]), actions: Cow::Borrowed(&[]), @@ -215,19 +272,6 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[ default: true, default_enabled: true, }, - PushRule { - rule_id: Cow::Borrowed("global/override/.m.rule.reaction"), - priority_class: 5, - conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch( - EventMatchCondition { - key: Cow::Borrowed("type"), - pattern: Cow::Borrowed("m.reaction"), - }, - ))]), - actions: Cow::Borrowed(&[]), - default: true, - default_enabled: true, - }, PushRule { rule_id: Cow::Borrowed("global/override/.m.rule.room.server_acl"), priority_class: 5, @@ -313,6 +357,22 @@ pub const BASE_APPEND_POSTCONTENT_RULES: &[PushRule] = &[ ]; pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[ + // Beeper change: this rule is moved down from override. This means room + // rules take precedence, so if you enable bot notifications (by modifying + // this rule) notifications will not be sent for muted rooms. + PushRule { + rule_id: Cow::Borrowed("global/underride/.m.rule.suppress_notices"), + priority_class: 1, + conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch( + EventMatchCondition { + key: Cow::Borrowed("content.msgtype"), + pattern: Cow::Borrowed("m.notice"), + }, + ))]), + actions: Cow::Borrowed(&[]), + default: true, + default_enabled: true, + }, PushRule { rule_id: Cow::Borrowed("global/underride/.m.rule.call"), priority_class: 1, @@ -663,6 +723,32 @@ pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[ default: true, default_enabled: true, }, + // Enable notifications for reactions to your own messages *in rooms with less + // than 20 members*. + PushRule { + rule_id: Cow::Borrowed("global/underride/.com.beeper.reaction"), + priority_class: 1, + conditions: Cow::Borrowed(&[ + Condition::Known(KnownCondition::EventMatch(EventMatchCondition { + key: Cow::Borrowed("type"), + pattern: Cow::Borrowed("m.reaction"), + })), + Condition::Known(KnownCondition::RoomMemberCount { + is: Some(Cow::Borrowed("<20")), + }), + Condition::Known(KnownCondition::RelatedEventMatchType( + RelatedEventMatchTypeCondition { + key: Cow::Borrowed("sender"), + pattern_type: Cow::Borrowed(&EventMatchPatternType::UserId), + rel_type: Cow::Borrowed("m.annotation"), + include_fallbacks: None, + }, + )), + ]), + actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]), + default: true, + default_enabled: true, + }, PushRule { rule_id: Cow::Borrowed("global/underride/.org.matrix.msc3930.rule.poll_start_one_to_one"), priority_class: 1, diff --git a/synapse/api/constants.py b/synapse/api/constants.py index eb9e6cc39b..e706dda97c 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -111,6 +111,7 @@ class LoginType: SSO: Final = "m.login.sso" DUMMY: Final = "m.login.dummy" REGISTRATION_TOKEN: Final = "m.login.registration_token" + JWT: Final = "org.matrix.login.jwt" # This is used in the `type` parameter for /register when called by @@ -350,6 +351,7 @@ class ReceiptTypes: READ: Final = "m.read" READ_PRIVATE: Final = "m.read.private" FULLY_READ: Final = "m.fully_read" + BEEPER_INBOX_DONE: Final = "com.beeper.inbox.done" class PublicRoomsFilterFields: diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 159cd44237..202fc60724 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -63,6 +63,7 @@ ApplicationServiceTransactionWorkerStore, ApplicationServiceWorkerStore, ) +from synapse.storage.databases.main.beeper import BeeperStore from synapse.storage.databases.main.censor_events import CensorEventsStore from synapse.storage.databases.main.client_ips import ClientIpWorkerStore from synapse.storage.databases.main.delayed_events import DelayedEventsStore @@ -86,6 +87,7 @@ from synapse.storage.databases.main.monthly_active_users import ( MonthlyActiveUsersWorkerStore, ) +from synapse.storage.databases.main.openid import OpenIdStore from synapse.storage.databases.main.presence import PresenceStore from synapse.storage.databases.main.profile import ProfileWorkerStore from synapse.storage.databases.main.purge_events import PurgeEventsStore @@ -169,6 +171,8 @@ class GenericWorkerStore( ExperimentalFeaturesStore, SlidingSyncStore, DelayedEventsStore, + BeeperStore, + OpenIdStore, ): # Properties that multiple storage classes define. Tell mypy what the # expected type is. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index c958a278fc..f652b7a4fa 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -617,3 +617,11 @@ def read_config( # MSC4455: Preview URL capability # Tracked in: https://github.com/element-hq/synapse/issues/19719 self.msc4452_enabled: bool = experimental.get("msc4452_enabled", False) + + self.beeper_user_notification_counts_enabled = experimental.get( + "beeper_user_notification_counts_enabled", + False, + ) + + # MSC4446: Allow moving the fully read marker backwards. + self.msc4446_enabled: bool = experimental.get("msc4446_enabled", False) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index adbede7f16..3c5104ec52 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -588,6 +588,11 @@ def _serialize_event( if delay_id is not None: d["unsigned"]["org.matrix.msc4140.delay_id"] = delay_id + # Beeper: include internal stream ordering as HS order unsigned hint + stream_ordering = getattr(e.internal_metadata, "stream_ordering", None) + if stream_ordering: + d["unsigned"]["com.beeper.hs.order"] = stream_ordering + # invite_room_state and knock_room_state are a list of stripped room state events # that are meant to provide metadata about a room to an invitee/knocker. They are # intended to only be included in specific circumstances, such as down sync, and diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1c8e108ea8..dcda1c1754 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -423,6 +423,10 @@ async def _get_available_ui_auth_types(self, user: UserID) -> Iterable[str]: ): ui_auth_types.add(LoginType.SSO) + # If JWT is enabled, allow user to re-authenticate with one + if self.hs.config.jwt.jwt_enabled: + ui_auth_types.add(LoginType.JWT) + return ui_auth_types def get_enabled_auth_types(self) -> Iterable[str]: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 2225466648..782084fe2c 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -191,6 +191,11 @@ def __init__(self, hs: "HomeServer"): desc="delete_stale_devices", func=self._delete_stale_devices, ) + # Beep: run this immediately since looping_call waits 24h after pod start + self.hs.run_as_background_process( + "delete_stale_devices", + self._delete_stale_devices, + ) async def _delete_stale_devices(self) -> None: """Background task that deletes devices which haven't been accessed for more than diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 2bc7efeb5e..501e82daa9 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -19,9 +19,11 @@ # # import logging +import time from typing import TYPE_CHECKING, cast import attr +from prometheus_client import Histogram from twisted.python.failure import Failure @@ -33,6 +35,7 @@ from synapse.handlers.relations import BundledAggregations from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.rest.admin._base import assert_user_is_admin from synapse.streams.config import PaginationConfig from synapse.types import ( @@ -55,6 +58,12 @@ logger = logging.getLogger(__name__) +purge_time = Histogram( + "room_purge_time", + "Time taken to purge rooms (sec)", + labelnames=[SERVER_NAME_LABEL], +) + # How many single event gaps we tolerate returning in a `/messages` response before we # backfill and try to fill in the history. This is an arbitrarily picked number so feel # free to tune it in the future. @@ -441,6 +450,7 @@ async def purge_room( room_id: room to be purged force: set true to skip checking for joined users. """ + purge_start = time.time() logger.info("starting purge room_id=%s force=%s", room_id, force) async with self._worker_locks.acquire_multi_read_write_lock( @@ -463,6 +473,10 @@ async def purge_room( await self._storage_controllers.purge_events.purge_room(room_id) + purge_end = time.time() + purge_time.labels(**{SERVER_NAME_LABEL: self.server_name}).observe( + purge_end - purge_start + ) logger.info("purge complete for room_id %s", room_id) @trace diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 85d2dd62bb..1adb5ac194 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -20,10 +20,11 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from synapse.api.constants import ReceiptTypes from synapse.api.errors import SynapseError +from synapse.types import JsonDict from synapse.util.async_helpers import Linearizer if TYPE_CHECKING: @@ -41,7 +42,12 @@ def __init__(self, hs: "HomeServer"): ) async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + allow_backward: bool = False, + extra_content: Optional[JsonDict] = None, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -59,7 +65,7 @@ async def received_client_read_marker( # Get event ordering, this also ensures we know about the event event_ordering = await self.store.get_event_ordering(event_id, room_id) - if existing_read_marker: + if existing_read_marker and not allow_backward: try: old_event_ordering = await self.store.get_event_ordering( existing_read_marker["event_id"], room_id @@ -73,7 +79,7 @@ async def received_client_read_marker( should_update = event_ordering > old_event_ordering if should_update: - content = {"event_id": event_id} + content = content = {"event_id": event_id, **(extra_content or {})} await self.account_data_handler.add_account_data_to_room( user_id, room_id, ReceiptTypes.FULLY_READ, content ) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f6383baf0b..879997f633 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -181,6 +181,7 @@ async def received_client_receipt( user_id: UserID, event_id: str, thread_id: str | None, + extra_content: JsonDict | None = None, ) -> None: """Called when a client tells us a local user has read up to the given event_id in the room. @@ -197,7 +198,7 @@ async def received_client_receipt( user_id=user_id.to_string(), event_ids=[event_id], thread_id=thread_id, - data={"ts": int(self.clock.time_msec())}, + data={"ts": int(self.clock.time_msec()), **(extra_content or {})}, ) is_new = await self._handle_new_receipts([receipt]) diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index ee4f8d672e..afbbb2d70c 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -432,9 +432,15 @@ async def _get_threads_for_events( return results - @trace + # Beep beep: bundled aggregations aren't used in Beeper clients and are thus a wasted calculation async def get_bundled_aggregations( self, filtered_events: Iterable[FilteredEvent], user_id: str + ) -> dict[str, BundledAggregations]: + return {} + + @trace + async def _orig_get_bundled_aggregations( + self, filtered_events: Iterable[FilteredEvent], user_id: str ) -> dict[str, BundledAggregations]: """Generate bundled aggregations for events. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13647caa2a..c2d1f1a2cb 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -26,6 +26,7 @@ import math import random import string +import time from collections import OrderedDict from collections.abc import Mapping from http import HTTPStatus @@ -38,6 +39,7 @@ ) import attr +from prometheus_client import Histogram import synapse.events.snapshot from synapse.api.constants import ( @@ -74,6 +76,7 @@ copy_and_fixup_power_levels_contents, ) from synapse.handlers.relations import BundledAggregations +from synapse.metrics import SERVER_NAME_LABEL from synapse.rest.admin._base import assert_user_is_admin from synapse.streams import EventSource from synapse.types import ( @@ -112,6 +115,23 @@ FIVE_MINUTES_IN_MS = 5 * 60 * 1000 +shutdown_time = Histogram( + "room_shutdown_time", + "Time taken to shutdown rooms (sec)", + labelnames=[SERVER_NAME_LABEL], +) +shutdown_kick_count = Histogram( + "room_shutdown_kick_count", + "Number of users successfully kicked while shutting down a room", + labelnames=[SERVER_NAME_LABEL], +) +shutdown_failed_kick_count = Histogram( + "room_shutdown_failed_kick_count", + "Number of users that were failed to be kicked while shutting down a room", + labelnames=[SERVER_NAME_LABEL], +) + + @attr.s(slots=True, frozen=True, auto_attribs=True) class EventContext: events_before: list[FilteredEvent] @@ -158,7 +178,8 @@ def __init__(self, hs: "HomeServer"): "history_visibility": HistoryVisibility.SHARED, "original_invitees_have_ops": True, "guest_can_join": True, - "power_level_content_override": {"invite": 0}, + # Beeper change: don't allow redactions by anyone in DM chats + "power_level_content_override": {"invite": 0, "redact": 1000}, }, RoomCreationPreset.PUBLIC_CHAT: { "join_rules": JoinRules.PUBLIC, @@ -432,6 +453,9 @@ async def _upgrade_room( additional_creators, ) + # Beeper: clear out any push actions and summaries for this room + await self.store.beeper_cleanup_tombstoned_room(old_room_id) + return new_room_id async def _update_upgraded_room_pls( @@ -2395,6 +2419,7 @@ async def shutdown_room( else: logger.info("Shutting down room %r", room_id) + shutdown_start = time.time() users = await self.store.get_local_users_related_to_room(room_id) for user_id, membership in users: # If the user is not in the room (or is banned), nothing to do. @@ -2482,4 +2507,15 @@ async def shutdown_room( else: result["local_aliases"] = [] + shutdown_end = time.time() + shutdown_kick_count.labels(**{SERVER_NAME_LABEL: self.hs.hostname}).observe( + len(result["kicked_users"]) + ) + shutdown_failed_kick_count.labels( + **{SERVER_NAME_LABEL: self.hs.hostname} + ).observe(len(result["failed_to_kick_users"])) + shutdown_time.labels(**{SERVER_NAME_LABEL: self.hs.hostname}).observe( + shutdown_end - shutdown_start + ) + return result diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9ecfe0da0f..b80603ccd4 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -119,6 +119,7 @@ class SyncConfig: is_guest: bool device_id: str | None use_state_after: bool + beeper_previews: bool = False @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -154,6 +155,7 @@ class JoinedSyncResult: unread_thread_notifications: JsonDict summary: JsonDict | None unread_count: int + preview: JsonDict | None def __bool__(self) -> bool: """Make the result appear empty if there are no updates. This is used @@ -165,6 +167,7 @@ def __bool__(self) -> bool: or self.ephemeral or self.account_data or self.sticky + or self.preview # nb the notification count does not, er, count: if there's nothing # else in the result, we don't need to send it. ) @@ -2800,6 +2803,8 @@ async def _generate_room_entry( } ) + user_id = sync_result_builder.sync_config.user.to_string() + # Note: `batch` can be both empty and limited here in the case where # `_load_filtered_recents` can't find any events the user should see # (e.g. due to having ignored the sender of the last 50 events). @@ -2809,7 +2814,6 @@ async def _generate_room_entry( # newly joined room, unless either a) they've joined before or b) the # tag was added by synapse e.g. for server notice rooms. if full_state: - user_id = sync_result_builder.sync_config.user.to_string() tags = await self.store.get_tags_for_room(user_id, room_id) # If there aren't any tags, don't send the empty tags list down @@ -2936,8 +2940,44 @@ async def _generate_room_entry( summary=summary, unread_count=0, sticky=sticky_events, + preview=None, ) + # Only generate previews if we have new events that would change it + if batch.events and sync_config.beeper_previews: + preview = ( + await self.store.beeper_preview_event_for_room_id_and_user_id( + room_id=room_id, user_id=user_id, to_key=now_token.room_key + ) + ) + + if preview: + preview_event_id, preview_origin_server_ts = preview + room_sync.preview = { + "event_id": preview_event_id, + "origin_server_ts": preview_origin_server_ts, + } + + # Check if we already have the event in the batch, in which + # case we needn't add it here. No point in checking state as + # we don't preview state events. + for ev in batch.events: + if ev.event.event_id == preview_event_id: + break + else: + preview_event = await self.store.get_event( + preview_event_id, + allow_none=True, + ) + if preview_event: + room_sync.preview["event"] = FilteredEvent( + event=preview_event, + membership=None, + ) + else: + # This should never happen! + logger.warning("Beeper preview is missing! roomID=%s", room_id) + if room_sync or always_include: notifs = await self.unread_notifs_for_room_id(room_id, sync_config) diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py index a0097dbc96..6846b3b272 100644 --- a/synapse/handlers/ui_auth/checkers.py +++ b/synapse/handlers/ui_auth/checkers.py @@ -27,6 +27,7 @@ from synapse.api.constants import LoginType from synapse.api.errors import Codes, LoginError, SynapseError +from synapse.types import UserID from synapse.util.json import json_decoder if TYPE_CHECKING: @@ -321,6 +322,87 @@ async def check_auth(self, authdict: dict, clientip: str) -> Any: ) +class JwtAuthChecker(UserInteractiveAuthChecker): + AUTH_TYPE = LoginType.JWT + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + self.hs = hs + + def is_enabled(self) -> bool: + return bool(self.hs.config.jwt.jwt_enabled) + + async def check_auth(self, authdict: dict, clientip: str) -> Any: + token = authdict.get("token", None) + if token is None: + raise LoginError( + 403, "Token field for JWT is missing", errcode=Codes.FORBIDDEN + ) + + from authlib.jose import JsonWebToken, JWTClaims + from authlib.jose.errors import BadSignatureError, InvalidClaimError, JoseError + + jwt = JsonWebToken([self.hs.config.jwt.jwt_algorithm]) + claim_options = {} + if self.hs.config.jwt.jwt_issuer is not None: + claim_options["iss"] = { + "value": self.hs.config.jwt.jwt_issuer, + "essential": True, + } + if self.hs.config.jwt.jwt_audiences is not None: + claim_options["aud"] = { + "values": self.hs.config.jwt.jwt_audiences, + "essential": True, + } + + try: + claims = jwt.decode( + token, + key=self.hs.config.jwt.jwt_secret, + claims_cls=JWTClaims, + claims_options=claim_options, + ) + except BadSignatureError: + # We handle this case separately to provide a better error message + raise LoginError( + 403, + "JWT validation failed: Signature verification failed", + errcode=Codes.FORBIDDEN, + ) + except JoseError as e: + # A JWT error occurred, return some info back to the client. + raise LoginError( + 403, + "JWT validation failed: %s" % (str(e),), + errcode=Codes.FORBIDDEN, + ) + + try: + claims.validate(leeway=120) # allows 2 min of clock skew + + # Enforce the old behavior which is rolled out in productive + # servers: if the JWT contains an 'aud' claim but none is + # configured, the login attempt will fail + if claims.get("aud") is not None: + if ( + self.hs.config.jwt.jwt_audiences is None + or len(self.hs.config.jwt.jwt_audiences) == 0 + ): + raise InvalidClaimError("aud") + except JoseError as e: + raise LoginError( + 403, + "JWT validation failed: %s" % (str(e),), + errcode=Codes.FORBIDDEN, + ) + + user = claims.get(self.hs.config.jwt.jwt_subject_claim, None) + if user is None: + raise LoginError(403, "Invalid JWT", errcode=Codes.FORBIDDEN) + + return UserID(user, self.hs.hostname).to_string() + + INTERACTIVE_AUTH_CHECKERS: Sequence[type[UserInteractiveAuthChecker]] = [ DummyAuthChecker, TermsAuthChecker, @@ -328,5 +410,6 @@ async def check_auth(self, authdict: dict, clientip: str) -> Any: EmailIdentityAuthChecker, MsisdnAuthChecker, RegistrationTokenAuthChecker, + JwtAuthChecker, ] """A list of UserInteractiveAuthChecker classes""" diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py index 15b92d7ef3..6268607087 100644 --- a/synapse/logging/__init__.py +++ b/synapse/logging/__init__.py @@ -22,10 +22,19 @@ import logging from synapse.logging._remote import RemoteHandler -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + BeeperTerseJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) # These are imported to allow for nicer logging configuration files. -__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"] +__all__ = [ + "RemoteHandler", + "JsonFormatter", + "TerseJsonFormatter", + "BeeperTerseJsonFormatter", +] # Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc issue9533_logger = logging.getLogger("synapse.9533_debug") diff --git a/synapse/logging/_terse_json.py b/synapse/logging/_terse_json.py index d9ff70b252..f8db186d4c 100644 --- a/synapse/logging/_terse_json.py +++ b/synapse/logging/_terse_json.py @@ -93,3 +93,15 @@ def format(self, record: logging.LogRecord) -> str: } return self._format(record, event) + + +class BeeperTerseJsonFormatter(JsonFormatter): + def format(self, record: logging.LogRecord) -> str: + event = { + "message": record.getMessage(), + "namespace": record.name, + "level": record.levelname.lower(), + "time": round(record.created, 2), + } + + return self._format(record, event) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 03dd341744..0a09600425 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -89,36 +89,36 @@ SENTINEL = object() -def _should_count_as_unread(event: EventBase, context: EventContext) -> bool: +def _should_count_as_unread( + event: EventBase, + context: EventContext, + non_bot_room_members_count: int, + current_user: str, + related_events: dict[str, dict[str, Any]], +) -> bool: # Exclude rejected and soft-failed events. if context.rejected or event.internal_metadata.is_soft_failed(): return False - # Exclude notices. - if ( - not event.is_state() - and event.type == EventTypes.Message - and event.content.get("msgtype") == "m.notice" - ): - return False - # Exclude edits. relates_to = relation_from_event(event) if relates_to and relates_to.rel_type == RelationTypes.REPLACE: return False - # Mark events that have a non-empty string body as unread. - body = event.content.get("body") - if isinstance(body, str) and body: - return True - - # Mark some state events as unread. - if event.is_state() and event.type in STATE_EVENT_TYPES_TO_MARK_UNREAD: - return True - - # Mark encrypted events as unread. - if not event.is_state() and event.type == EventTypes.Encrypted: - return True + # Mark encrypted and plain text messages events as unread. + if not event.is_state(): + if event.type == EventTypes.Encrypted: + return True + elif event.type == EventTypes.Message: + body = event.content.get("body") + return isinstance(body, str) and bool(body) + # Beeper: We want reactions to only count as unread if they're reactions to the current user in rooms that + # have fewer than 20 users. + elif event.type == "m.reaction" and related_events.get("m.annotation"): + return ( + related_events["m.annotation"]["sender"] == current_user + and non_bot_room_members_count < 20 + ) return False @@ -380,15 +380,9 @@ async def _action_for_event_by_user( # (historical messages persisted in reverse-chronological order). return - # Disable counting as unread unless the experimental configuration is - # enabled, as it can cause additional (unwanted) rows to be added to the - # event_push_actions table. - count_as_unread = False - if self.hs.config.experimental.msc2654_enabled: - count_as_unread = _should_count_as_unread(event, context) - rules_by_user = await self._get_rules_for_event(event) actions_by_user: dict[str, Collection[Mapping | str]] = {} + count_as_unread_by_user: dict[str, bool] = {} # Gather a bunch of info in parallel. # @@ -501,12 +495,19 @@ async def _action_for_event_by_user( if not isinstance(display_name, str): display_name = None - if count_as_unread: - # Add an element for the current user if the event needs to be marked as - # unread, so that add_push_actions_to_staging iterates over it. - # If the event shouldn't be marked as unread but should notify the - # current user, it'll be added to the dict later. - actions_by_user[uid] = [] + # Beeper: Need to calculate this per user as whether it should count as unread or not + # depends on who the current user is. + if self.hs.config.experimental.msc2654_enabled: + count_as_unread_by_user[uid] = _should_count_as_unread( + event, context, room_member_count, uid, related_events + ) + + if count_as_unread_by_user[uid]: + # Add an element for the current user if the event needs to be marked as + # unread, so that add_push_actions_to_staging iterates over it. + # If the event shouldn't be marked as unread but should notify the + # current user, it'll be added to the dict later. + actions_by_user[uid] = [] msc4306_thread_subscription_state: bool | None = None if msc4306_thread_subscribers is not None: @@ -545,7 +546,7 @@ async def _action_for_event_by_user( await self.store.add_push_actions_to_staging( event.event_id, actions_by_user, - count_as_unread, + count_as_unread_by_user, thread_id, ) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ca63a99e3e..a08a4da840 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -143,6 +143,8 @@ def __init__(self, hs: "HomeServer", pusher_config: PusherConfig): pusher_config.app_id, pusher_config.pushkey, ) + # Beeper: Save this so we can pass this on to Sygnal as well + self.user_name = pusher_config.user_name # Validate that there's a URL and it is of the proper form. if "url" not in self.data: @@ -469,7 +471,7 @@ async def dispatch_push_event( rejected push keys otherwise. If this array is empty, the push fully succeeded. """ - priority = "low" + priority = "high" # Beep: always use high priority if ( event.type == EventTypes.Encrypted or tweaks.get("highlight") @@ -485,10 +487,14 @@ async def dispatch_push_event( content: JsonDict = { "event_id": event.event_id, "room_id": event.room_id, + "com.beeper.user_id": self.user_id, "prio": priority, } if not self.disable_badge_count: - content["counts"] = {"unread": badge} + content["counts"] = { + "unread": badge, + "com.beeper.server_type": "synapse", + } # event_id_only doesn't include the tweaks, so override them. tweaks = {} else: @@ -503,10 +509,12 @@ async def dispatch_push_event( "type": event.type, "sender": event.sender, "prio": priority, + "com.beeper.user_id": self.user_id, } if not self.disable_badge_count: content["counts"] = { "unread": badge, + "com.beeper.server_type": "synapse", } if event.type == "m.room.member" and event.is_state(): content["membership"] = event.content["membership"] @@ -540,7 +548,11 @@ async def _send_badge(self, badge: int) -> None: "id": "", "type": None, "sender": "", - "counts": {"unread": badge}, + "counts": { + "unread": badge, + "com.beeper.server_type": "synapse", + }, + "com.beeper.user_id": self.user_id, "devices": [ { "app_id": self.app_id, diff --git a/synapse/res/templates/_base.html b/synapse/res/templates/_base.html index 4b5cc7bcb6..b36d316a13 100644 --- a/synapse/res/templates/_base.html +++ b/synapse/res/templates/_base.html @@ -12,15 +12,7 @@
- {% if app_name == "Riot" %} - [Riot] - {% elif app_name == "Vector" %} - [Vector] - {% elif app_name == "Element" %} - [Element] - {% else %} - [matrix] - {% endif %} + [Beeper]
{% block body %}{% endblock %} diff --git a/synapse/res/templates/password_reset.html b/synapse/res/templates/password_reset.html index 1f267946c8..0b0c969554 100644 --- a/synapse/res/templates/password_reset.html +++ b/synapse/res/templates/password_reset.html @@ -2,9 +2,9 @@ {% block title %}Password reset{% endblock %} {% block body %} -

A password reset request has been received for your Matrix account. If this was you, please click the link below to confirm resetting your password:

+

A password reset request has been received for your Beeper account. If this was you, please click the link below to confirm resetting your password:

{{ link }} -

If this was not you, do not click the link above and instead contact your server administrator. Thank you.

+

If this was not you, do not click the link above and instead contact the Beeper Support team. Thank you.

{% endblock %} diff --git a/synapse/res/templates/password_reset_confirmation.html b/synapse/res/templates/password_reset_confirmation.html index fabb9a6ed5..6af2d5aa7c 100644 --- a/synapse/res/templates/password_reset_confirmation.html +++ b/synapse/res/templates/password_reset_confirmation.html @@ -8,7 +8,7 @@ -

You have requested to reset your Matrix account password. Click the link below to confirm this action.

+

You have requested to reset your Beeper account password. Click the link below to confirm this action.

If you did not mean to do this, please close this page and your password will not be changed.

diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 0774b6ed40..106bb09d0e 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -289,6 +289,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: else: UserRestServletV2Get(hs).register(http_server) + # Beep: except this one ;) + UserTokenRestServlet(hs).register(http_server) return auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 3783211a92..f4893d190e 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -19,11 +19,13 @@ # # import logging +import time from http import HTTPStatus from typing import TYPE_CHECKING, cast import attr from immutabledict import immutabledict +from prometheus_client import Histogram from synapse.api.constants import Direction, EventTypes, JoinRules, Membership from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError @@ -50,6 +52,7 @@ ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.rest.admin._base import ( admin_patterns, assert_requester_is_admin, @@ -69,6 +72,12 @@ logger = logging.getLogger(__name__) +delete_time = Histogram( + "admin_room_delete_time", + "Time taken to delete rooms via the admin API (sec)", + labelnames=[SERVER_NAME_LABEL], +) + class AdminRoomHierarchy(RestServlet): """ @@ -399,13 +408,21 @@ async def on_GET( async def on_DELETE( self, request: SynapseRequest, room_id: str ) -> tuple[int, JsonDict]: - return await self._delete_room( + logger.info(f"[admin/rooms] deleting {room_id}") + start = time.time() + response = await self._delete_room( request, room_id, self.auth, self.room_shutdown_handler, self.pagination_handler, ) + end = time.time() + logger.info(f"[admin/rooms] deleting {room_id} took {end - start} seconds") + delete_time.labels(**{SERVER_NAME_LABEL: request.our_server_name}).observe( + end - start + ) + return response async def _delete_room( self, diff --git a/synapse/rest/client/account_data.py b/synapse/rest/client/account_data.py index b18232fc56..841c02f9f7 100644 --- a/synapse/rest/client/account_data.py +++ b/synapse/rest/client/account_data.py @@ -19,6 +19,7 @@ # # +import json import logging from typing import TYPE_CHECKING @@ -27,6 +28,7 @@ from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest +from synapse.rest.client.read_marker import ReadMarkerRestServlet from synapse.types import JsonDict, JsonMapping, RoomID from ._base import client_patterns @@ -310,9 +312,113 @@ async def on_DELETE( return 200, {} +class RoomBeeperInboxStateServlet(RestServlet): + """ + PUT /user/{user_id}/rooms/{room_id}/beeper_inbox_state HTTP/1.1 + """ + + PATTERNS = list( + client_patterns( + "/com.beeper.inbox/user/(?P[^/]*)/rooms/(?P[^/]*)/inbox_state", + releases=(), # not in the matrix spec, only include under /unstable + ) + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.clock = hs.get_clock() + self.store = hs.get_datastores().main + self.handler = hs.get_account_data_handler() + self.read_marker_client = ReadMarkerRestServlet(hs) + + async def on_PUT( + self, request: SynapseRequest, user_id: str, room_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): + raise AuthError(403, "Cannot add beeper inbox state for other users.") + + if not RoomID.is_valid(room_id): + raise SynapseError( + 400, + f"{room_id} is not a valid room ID", + Codes.INVALID_PARAM, + ) + + ts = self.clock.time_msec() + + body = parse_json_object_from_request(request) + + if "done" in body: + delta_ms = body["done"].get("at_delta") or 0 + done = {"updated_ts": ts, "at_ts": ts + delta_ms} + if "at_order" in body["done"]: + done["at_order"] = body["done"]["at_order"] + await self.handler.add_account_data_to_room( + user_id, room_id, "com.beeper.inbox.done", done + ) + logger.info( + f"SetBeeperDone done_delta_ms={delta_ms} at_order={body.get('at_order')}" + ) + + if "marked_unread" in body: + marked_unread = {"unread": body["marked_unread"], "ts": ts} + await self.handler.add_account_data_to_room( + user_id, room_id, "m.marked_unread", marked_unread + ) + logger.info(f"SetBeeperMarkedUnread marked_unread={body['marked_unread']}") + + if "read_markers" in body: + await self.read_marker_client.handle_read_marker( + room_id, body["read_markers"], requester + ) + logger.info( + f"SetBeeperReadMarkers read_markers={json.dumps(body['read_markers'])}" + ) + + return 200, {} + + +class BeeperInboxBatchArchiveServlet(RestServlet): + """ + PUT /com.beeper.inbox/batch_archive HTTP/1.1 + """ + + PATTERNS = list( + client_patterns( + "/com.beeper.inbox/batch_archive", + releases=(), # not in the matrix spec, only include under /unstable + ) + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.clock = hs.get_clock() + self.store = hs.get_datastores().main + self.handler = hs.get_account_data_handler() + + async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + ts = self.clock.time_msec() + body = parse_json_object_from_request(request) + + done = {"updated_ts": ts, "at_ts": ts} + for room_id in body["room_ids"]: + # TODO in transaction + await self.handler.add_account_data_to_room( + requester.user.to_string(), room_id, "com.beeper.inbox.done", done + ) + + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: AccountDataServlet(hs).register(http_server) RoomAccountDataServlet(hs).register(http_server) + RoomBeeperInboxStateServlet(hs).register(http_server) + BeeperInboxBatchArchiveServlet(hs).register(http_server) if hs.config.experimental.msc3391_enabled: UnstableAccountDataServlet(hs).register(http_server) diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index f80a43b297..01c93e2bc1 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -88,6 +88,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: [ ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, + ReceiptTypes.BEEPER_INBOX_DONE, ], ) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index 874e7487bf..2331c3a2af 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -23,10 +23,11 @@ from typing import TYPE_CHECKING from synapse.api.constants import ReceiptTypes +from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest -from synapse.types import JsonDict +from synapse.types import JsonDict, Requester from ._base import client_patterns @@ -58,14 +59,32 @@ async def on_POST( self, request: SynapseRequest, room_id: str ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) + body = parse_json_object_from_request(request) + return await self.handle_read_marker(room_id, body, requester) + # Beeper: The endpoint and underlying method are separated here so `inbox_state` + # can use the same function. + async def handle_read_marker( + self, room_id: str, body: dict, requester: Requester + ) -> tuple[int, JsonDict]: await self.presence_handler.bump_presence_active_time( requester.user, requester.device_id ) - body = parse_json_object_from_request(request) - unrecognized_types = set(body.keys()) - self._known_receipt_types + + if self.config.experimental.msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + 400, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + unrecognized_types -= {"com.beeper.allow_backward"} + else: + allow_backward = False + if unrecognized_types: # It's fine if there are unrecognized receipt types, but let's log # it to help debug clients that have typoed the receipt type. @@ -86,6 +105,8 @@ async def on_POST( room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, + extra_content=body.get("com.beeper.fully_read.extra", None), ) else: await self.receipts_handler.received_client_receipt( @@ -95,6 +116,7 @@ async def on_POST( event_id=event_id, # Setting the thread ID is not possible with the /read_markers endpoint. thread_id=None, + extra_content=body.get("com.beeper.read.extra", None), ) return 200, {} diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index d3a43537bb..52c876d2d5 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -20,6 +20,7 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes @@ -50,11 +51,13 @@ def __init__(self, hs: "HomeServer"): self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() self._main_store = hs.get_datastores().main + self._msc4446_enabled = hs.config.experimental.msc4446_enabled self._known_receipt_types = { ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, ReceiptTypes.FULLY_READ, + ReceiptTypes.BEEPER_INBOX_DONE, } async def on_POST( @@ -71,7 +74,26 @@ async def on_POST( f"Receipt type must be {', '.join(self._known_receipt_types)}", ) - body = parse_json_object_from_request(request) + body = parse_json_object_from_request(request, allow_empty_body=False) + + if self._msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + if allow_backward and receipt_type != ReceiptTypes.FULLY_READ: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward is only allowed to be true for " + f"{ReceiptTypes.FULLY_READ}.", + Codes.INVALID_PARAM, + ) + else: + allow_backward = False # Pull the thread ID, if one exists. thread_id = None @@ -108,6 +130,8 @@ async def on_POST( room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, + extra_content=body, ) else: await self.receipts_handler.received_client_receipt( @@ -116,6 +140,7 @@ async def on_POST( user_id=requester.user, event_id=event_id, thread_id=thread_id, + extra_content=body, ) return 200, {} diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 73832ba7a8..da3bc81228 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -367,10 +367,11 @@ def __init__(self, hs: "HomeServer"): ) async def on_GET(self, request: Request) -> tuple[int, JsonDict]: - if not self.hs.config.registration.enable_registration: - raise SynapseError( - 403, "Registration has been disabled", errcode=Codes.FORBIDDEN - ) + # Beeper: We should be able to check availability of usernames even though public registration is disabled + # if not self.hs.config.registration.enable_registration: + # raise SynapseError( + # 403, "Registration has been disabled", errcode=Codes.FORBIDDEN + # ) if self.inhibit_user_in_use_error: return 200, {"available": True} diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index c3cf0dc3c4..c5337d3e04 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -165,6 +165,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: ) filter_id = parse_string(request, "filter") full_state = parse_boolean(request, "full_state", default=False) + beeper_previews = parse_boolean(request, "beeper_previews", default=False) use_state_after = False if await self.store.is_feature_enabled( @@ -176,13 +177,14 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: logger.debug( "/sync: user=%r, timeout=%r, since=%r, " - "set_presence=%r, filter_id=%r, device_id=%r", + "set_presence=%r, filter_id=%r, device_id=%r, beeper_previews=%r", user, timeout, since, set_presence, filter_id, device_id, + beeper_previews, ) # Stream position of the last ignored users account data event for this user, @@ -207,6 +209,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: device_id, last_ignore_accdata_streampos, use_state_after, + beeper_previews, ) if filter_id is None: @@ -244,6 +247,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: is_guest=requester.is_guest, device_id=device_id, use_state_after=use_state_after, + beeper_previews=beeper_previews, ) since_token = None @@ -653,6 +657,17 @@ async def encode_room( if self._msc2654_enabled: result["org.matrix.msc2654.unread_count"] = room.unread_count + if room.preview: + if "event" in room.preview: + room.preview["event"] = ( + await self._event_serializer.serialize_events( + [room.preview["event"]], + time_now, + config=serialize_options, + ) + )[0] + result["com.beeper.inbox.preview"] = room.preview + return result diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index bb1711f2cf..8d3cbd8ce4 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -200,6 +200,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "org.matrix.msc4155": self.config.experimental.msc4155_enabled, # MSC4306: Support for thread subscriptions "org.matrix.msc4306": self.config.experimental.msc4306_enabled, + # MSC4446: Allow moving the fully read marker backwards. + "com.beeper.msc4446": self.config.experimental.msc4446_enabled, # MSC4169: Backwards-compatible redaction sending using `/send` "com.beeper.msc4169": self.config.experimental.msc4169_enabled, # MSC4354: Sticky events diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index 801d474ecc..66e2f9a928 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -52,6 +52,9 @@ async def get_well_known(self) -> JsonDict | None: result["m.identity_server"] = { "base_url": self._config.registration.default_identity_server } + else: + # Workaround for iOS expecting some value here + result["m.identity_server"] = {"base_url": ""} if self._config.mas.enabled: assert isinstance(self._auth, MasDelegatedAuth) diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 9f8d4debbe..b04e874505 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -44,6 +44,7 @@ from .account_data import AccountDataStore from .appservice import ApplicationServiceStore, ApplicationServiceTransactionStore +from .beeper import BeeperStore from .cache import CacheInvalidationWorkerStore from .censor_events import CensorEventsStore from .client_ips import ClientIpWorkerStore @@ -165,6 +166,7 @@ class DataStore( TaskSchedulerWorkerStore, SlidingSyncStore, DelayedEventsStore, + BeeperStore, ): def __init__( self, diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index 538280137b..f4706487ad 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -20,6 +20,7 @@ # import logging +import re from typing import ( TYPE_CHECKING, Any, @@ -57,6 +58,10 @@ logger = logging.getLogger(__name__) +# Regex pattern for detecting a bridge bot (cached here for performance) +SYNAPSE_BOT_PATTERN = re.compile(r"^@_.*_bot\:*") +HUNGRYSERV_BOT_PATTERN = re.compile(r"^@[a-z]+bot\:beeper.local") + class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore): def __init__( @@ -779,6 +784,18 @@ def _add_account_data_for_user( ) -> None: content_json = json_encoder.encode(content) + # If we're ignoring users, silently filter out any bots that may be ignored + if account_data_type == AccountDataTypes.IGNORED_USER_LIST: + ignored_users = content.get("ignored_users", {}) + if isinstance(ignored_users, dict): + content["ignored_users"] = { + u: v + for u, v in ignored_users.items() + if not ( + SYNAPSE_BOT_PATTERN.match(u) or HUNGRYSERV_BOT_PATTERN.match(u) + ) + } + self.db_pool.simple_upsert_txn( txn, table="account_data", diff --git a/synapse/storage/databases/main/beeper.py b/synapse/storage/databases/main/beeper.py new file mode 100644 index 0000000000..fea1cb3daf --- /dev/null +++ b/synapse/storage/databases/main/beeper.py @@ -0,0 +1,289 @@ +# Beep beep! + +import logging +from typing import TYPE_CHECKING, Optional, cast + +from synapse.events import EventBase +from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.storage._base import SQLBaseStore +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) +from synapse.types import RoomStreamToken +from synapse.util.duration import Duration + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class BeeperStore(SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.database = database + + self.user_notification_counts_enabled: bool = ( + hs.config.experimental.beeper_user_notification_counts_enabled + ) + + if ( + self.user_notification_counts_enabled + and hs.config.worker.run_background_tasks + ): + self.aggregate_notification_counts_loop = self.clock.looping_call( + self.beeper_aggregate_notification_counts, Duration(seconds=30) + ) + self.is_aggregating_notification_counts = False + + async def beeper_preview_event_for_room_id_and_user_id( + self, room_id: str, user_id: str, to_key: RoomStreamToken + ) -> Optional[tuple[str, int]]: + def beeper_preview_txn(txn: LoggingTransaction) -> Optional[tuple[str, int]]: + sql = """ + WITH latest_event AS ( + SELECT e.event_id, e.origin_server_ts + FROM events AS e + LEFT JOIN redactions as r + ON e.event_id = r.redacts + -- Look to see if this event itself is an edit, as we don't want to + -- use edits ever as the "latest event" + LEFT JOIN event_relations as is_edit + ON e.event_id = is_edit.event_id AND is_edit.relation_type = 'm.replace' + WHERE + e.stream_ordering <= ? + AND e.room_id = ? + AND is_edit.event_id IS NULL + AND r.redacts IS NULL + AND e.type IN ( + 'm.room.message', + 'm.room.encrypted', + 'm.reaction', + 'm.sticker' + ) + AND CASE + -- Only find non-redacted reactions to our own messages + WHEN (e.type = 'm.reaction') THEN ( + SELECT ? = ee.sender AND ee.event_id NOT IN ( + SELECT redacts FROM redactions WHERE redacts = ee.event_id + ) FROM events as ee + WHERE ee.event_id = ( + SELECT eer.relates_to_id FROM event_relations AS eer + WHERE eer.event_id = e.event_id + ) + ) + ELSE (true) END + ORDER BY e.stream_ordering DESC + LIMIT 1 + ), + latest_edit_for_latest_event AS ( + SELECT e.event_id, e_replacement.event_id as replacement_event_id + FROM latest_event e + -- Find any events that edit this event, as we'll want to use the new content from + -- the edit as the preview + LEFT JOIN event_relations as er + ON e.event_id = er.relates_to_id AND er.relation_type = 'm.replace' + LEFT JOIN events as e_replacement + ON er.event_id = e_replacement.event_id + ORDER BY e_replacement.origin_server_ts DESC + LIMIT 1 + ) + SELECT COALESCE(lefle.replacement_event_id, le.event_id), le.origin_server_ts + FROM latest_event le + LEFT JOIN latest_edit_for_latest_event lefle ON le.event_id = lefle.event_id + """ + + txn.execute( + sql, + ( + to_key.stream, + room_id, + user_id, + ), + ) + + return cast(Optional[tuple[str, int]], txn.fetchone()) + + return await self.db_pool.runInteraction( + "beeper_preview_for_room_id_and_user_id", + beeper_preview_txn, + ) + + async def beeper_cleanup_tombstoned_room(self, room_id: str) -> None: + def beeper_cleanup_tombstoned_room_txn(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_txn( + txn, table="event_push_actions", keyvalues={"room_id": room_id} + ) + self.db_pool.simple_delete_txn( + txn, table="event_push_summary", keyvalues={"room_id": room_id} + ) + + await self.db_pool.runInteraction( + "beeper_cleanup_tombstoned_room", + beeper_cleanup_tombstoned_room_txn, + ) + + def beeper_add_notification_counts_txn( + self, + txn: LoggingTransaction, + notifiable_events: list[EventBase], + ) -> None: + if not self.user_notification_counts_enabled: + return + + sql = """ + INSERT INTO beeper_user_notification_counts ( + room_id, event_stream_ordering, + user_id, thread_id, notifs, unreads, highlights + ) + SELECT ?, ?, user_id, thread_id, notif, unread, highlight + FROM event_push_actions_staging + WHERE event_id = ? + """ + + txn.execute_batch( + sql, + ( + ( + event.room_id, + event.internal_metadata.stream_ordering, + event.event_id, + ) + for event in notifiable_events + ), + ) + + def beeper_clear_notification_counts_txn( + self, + txn: LoggingTransaction, + user_id: str, + room_id: str, + stream_ordering: int, + ) -> None: + if not self.user_notification_counts_enabled: + return + + sql = """ + DELETE FROM beeper_user_notification_counts + WHERE + user_id = ? + AND room_id = ? + AND event_stream_ordering <= ? + """ + + txn.execute(sql, (user_id, room_id, stream_ordering)) + + @wrap_as_background_process("beeper_aggregate_notification_counts") + async def beeper_aggregate_notification_counts(self) -> None: + if not self.user_notification_counts_enabled: + return + + def aggregate_txn(txn: LoggingTransaction, limit: int) -> int: + sql = """ + WITH recent_rows AS ( -- Aggregate the tables, flag aggregated rows for deletion + SELECT + user_id, + room_id + FROM + beeper_user_notification_counts + WHERE + event_stream_ordering > ( + SELECT event_stream_ordering FROM beeper_user_notification_counts_stream_ordering + ) + AND event_stream_ordering < ( + -- Select highest stream ordering from events over one hour, + -- this is to avoid serialization issues with the most + -- recent events/receipts + SELECT stream_ordering FROM events + WHERE origin_server_ts < ( + (EXTRACT(EPOCH from NOW()) - 3600) * 1000 + ) + ORDER BY stream_ordering DESC + LIMIT 1 + ) + -- Oldest first, to reduce serialization issues + ORDER BY event_stream_ordering ASC + LIMIT {limit} + ) + UPDATE + beeper_user_notification_counts AS epc + SET + unreads = CASE WHEN epc.event_stream_ordering = agg.max_eso THEN agg.unreads ELSE 0 END, + notifs = CASE WHEN epc.event_stream_ordering = agg.max_eso THEN agg.notifs ELSE 0 END, + highlights = CASE WHEN epc.event_stream_ordering = agg.max_eso THEN agg.highlights ELSE 0 END, + aggregated = epc.event_stream_ordering != agg.max_eso + FROM ( + SELECT + user_id, + room_id, + SUM(unreads) AS unreads, + SUM(notifs) AS notifs, + SUM(highlights) AS highlights, + MAX(event_stream_ordering) AS max_eso + FROM + beeper_user_notification_counts + WHERE + user_id IN(SELECT user_id FROM recent_rows) + AND room_id IN(SELECT room_id FROM recent_rows) + GROUP BY + user_id, + room_id + ) AS agg + WHERE + epc.room_id = agg.room_id + AND epc.user_id = agg.user_id + RETURNING + event_stream_ordering; + """.format(limit=limit) + + txn.execute(sql) + orders = list(txn) + if not orders: + logger.info("No user counts aggregated") + return 0 + + max_stream_ordering = max(orders) + txn.execute( + """ + UPDATE beeper_user_notification_counts_stream_ordering + SET event_stream_ordering = ? + """, + (max_stream_ordering,), + ) + txn.execute("DELETE FROM beeper_user_notification_counts WHERE aggregated") + + logger.info(f"Aggregated {len(orders)} notification count rows") + + return txn.rowcount + + if self.is_aggregating_notification_counts: + return + + self.is_aggregating_notification_counts = True + limit = 1000 + + try: + logger.info("Aggregating notification counts") + + last_batch = limit + 1 + while last_batch > limit: + last_batch = await self.db_pool.runInteraction( + "beeper_aggregate_notification_counts", + aggregate_txn, + limit=limit, + ) + await self.clock.sleep(Duration(seconds=1)) + + except self.database.engine.module.OperationalError: + logger.exception("Failed to aggregate notifications") + + finally: + self.is_aggregating_notification_counts = False diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 7cd3667a2b..6844ed1030 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -20,6 +20,7 @@ # import logging +from os import environ from typing import ( TYPE_CHECKING, Mapping, @@ -51,8 +52,10 @@ # Number of msec of granularity to store the user IP 'last seen' time. Smaller # times give more inserts into the database even for readonly API hits -# 120 seconds == 2 minutes -LAST_SEEN_GRANULARITY = 120 * 1000 +# 120 seconds == 2 minutes, Beep: updated to 1h +LAST_SEEN_GRANULARITY = 3600 * 1000 + +DISABLE_CLIENT_IP_STORAGE = environ.get("SYNAPSE_DISABLE_CLIENT_IP_STORAGE") == "true" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -694,14 +697,16 @@ def _update_client_ips_batch_txn( devices_keys.append((user_id, device_id)) devices_values.append((user_agent, last_seen, ip)) - self.db_pool.simple_upsert_many_txn( - txn, - table="user_ips", - key_names=("user_id", "access_token", "ip"), - key_values=user_ips_keys, - value_names=("user_agent", "device_id", "last_seen"), - value_values=user_ips_values, - ) + # Beep: only store user_ips if not disabled + if not DISABLE_CLIENT_IP_STORAGE: + self.db_pool.simple_upsert_many_txn( + txn, + table="user_ips", + key_names=("user_id", "access_token", "ip"), + key_values=user_ips_keys, + value_names=("user_agent", "device_id", "last_seen"), + value_values=user_ips_values, + ) if devices_values: self.db_pool.simple_update_many_txn( diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index a66caa672c..b330c372b4 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -1156,7 +1156,7 @@ async def add_push_actions_to_staging( self, event_id: str, user_id_actions: dict[str, Collection[Mapping | str]], - count_as_unread: bool, + count_as_unread_by_user: dict[str, bool], thread_id: str, ) -> None: """Add the push actions for the event to the push action staging area. @@ -1184,7 +1184,7 @@ def _gen_entry( _serialize_action(actions, bool(is_highlight)), # actions column notif, # notif column is_highlight, # highlight column - int(count_as_unread), # unread column + int(count_as_unread_by_user.get(user_id, 0)), # unread column thread_id, # thread_id column self.clock.time_msec(), # inserted_ts column ) diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 5aab0067fc..91c923a78f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -3483,6 +3483,8 @@ def _set_push_actions_for_event_and_users_txn( ], ) + self.store.beeper_add_notification_counts_txn(txn, notifiable_events) + # Now we delete the staging area for *all* events that were being # persisted. txn.execute_batch( diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index fe8079c201..6aacf68f0e 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -453,7 +453,6 @@ def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> None: (room_id,), ) - if isinstance(self.database_engine, PostgresEngine): # Disable statement timeouts for this transaction; purging rooms can # take a while! txn.execute("SET LOCAL statement_timeout = 0") diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index ba5e07a051..4b27f7689b 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -978,6 +978,11 @@ def _insert_linearized_receipt_txn( where_clause=where_clause, ) + if self.hs.is_mine_id(user_id): + self.beeper_clear_notification_counts_txn( # type: ignore[attr-defined] + txn, user_id, room_id, stream_ordering + ) + return rx_ts def _graph_to_linear( diff --git a/synapse/storage/schema/main/delta/73/99_beeper_user_notification_counts.sql b/synapse/storage/schema/main/delta/73/99_beeper_user_notification_counts.sql new file mode 100644 index 0000000000..674cc4d3ec --- /dev/null +++ b/synapse/storage/schema/main/delta/73/99_beeper_user_notification_counts.sql @@ -0,0 +1,19 @@ +CREATE TABLE beeper_user_notification_counts ( + user_id TEXT, + room_id TEXT, + thread_id TEXT, + event_stream_ordering BIGINT, + notifs BIGINT, + unreads BIGINT, + highlights BIGINT, + aggregated BOOLEAN, + UNIQUE (user_id, room_id, thread_id, event_stream_ordering) +); + +CREATE TABLE beeper_user_notification_counts_stream_ordering ( + lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + event_stream_ordering BIGINT NOT NULL, + CHECK (lock='X') +); + +INSERT INTO beeper_user_notification_counts_stream_ordering (event_stream_ordering) VALUES (0); diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index c1790fd3ae..f27bb8bc24 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -101,12 +101,14 @@ class TaskScheduler: # Time before a complete or failed task is deleted from the DB KEEP_TASKS_FOR_MS = 7 * 24 * 60 * 60 * 1000 # 1 week # Maximum number of tasks that can run at the same time - MAX_CONCURRENT_RUNNING_TASKS = 5 + MAX_CONCURRENT_RUNNING_TASKS = 2 # Beep: temporarily changed from 5 # Time from the last task update after which we will log a warning LAST_UPDATE_BEFORE_WARNING_MS = 24 * 60 * 60 * 1000 # 24hrs # Report a running task's status and usage every so often. OCCASIONAL_REPORT_INTERVAL = Duration(minutes=5) + SLEEP_AFTER_TASK_S = 1 + def __init__(self, hs: "HomeServer"): self.hs = hs # nb must be called this for @wrap_as_background_process self.server_name = hs.hostname @@ -487,6 +489,12 @@ async def wrapper() -> None: status = TaskStatus.FAILED error = f.getErrorMessage() + # Beep: sleep before we remove/complete the running task + if TaskScheduler.SLEEP_AFTER_TASK_S > 0: + await self._clock.sleep( + Duration(seconds=TaskScheduler.SLEEP_AFTER_TASK_S) + ) + await self._store.update_scheduled_task( task.id, self._clock.time_msec(), diff --git a/tests/push/test_bulk_push_rule_evaluator.py b/tests/push/test_bulk_push_rule_evaluator.py index 137bbe24b2..1a21d00385 100644 --- a/tests/push/test_bulk_push_rule_evaluator.py +++ b/tests/push/test_bulk_push_rule_evaluator.py @@ -436,7 +436,7 @@ def test_suppress_edits(self) -> None: ) # An edit which is a mention will cause a notification. - self.assertTrue( + self.assertFalse( # Beeper: changed from true per our base rule changes self._create_and_process( bulk_evaluator, { diff --git a/tests/push/test_http.py b/tests/push/test_http.py index ca2ced01ed..8d73a436b1 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -419,8 +419,8 @@ def test_sends_high_priority_for_one_to_one_only(self) -> None: self.push_attempts[1][1], "http://example.com/_matrix/push/v1/notify" ) - # check that this is low-priority - self.assertEqual(self.push_attempts[1][2]["notification"]["prio"], "low") + # Beeper: all notifications are high priority + self.assertEqual(self.push_attempts[1][2]["notification"]["prio"], "high") def test_sends_high_priority_for_mention(self) -> None: """ @@ -496,8 +496,8 @@ def test_sends_high_priority_for_mention(self) -> None: self.push_attempts[1][1], "http://example.com/_matrix/push/v1/notify" ) - # check that this is low-priority - self.assertEqual(self.push_attempts[1][2]["notification"]["prio"], "low") + # Beeper: all notifications are high priority + self.assertEqual(self.push_attempts[1][2]["notification"]["prio"], "high") def test_sends_high_priority_for_atroom(self) -> None: """ @@ -580,8 +580,8 @@ def test_sends_high_priority_for_atroom(self) -> None: self.push_attempts[1][1], "http://example.com/_matrix/push/v1/notify" ) - # check that this is low-priority - self.assertEqual(self.push_attempts[1][2]["notification"]["prio"], "low") + # Beeper: all notifications are high priority + self.assertEqual(self.push_attempts[1][2]["notification"]["prio"], "high") def test_push_unread_count_group_by_room(self) -> None: """ diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index 2e389710b9..407594df4f 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -27,13 +27,14 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService +from synapse.events import FrozenEvent from synapse.push.bulk_push_rule_evaluator import _flatten_dict from synapse.push.httppusher import tweaks_for_actions from synapse.rest import admin from synapse.rest.client import login, register, room from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.synapse_rust.push import PushRuleEvaluator +from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator, PushRules from synapse.types import JsonDict, JsonMapping, UserID from synapse.util.clock import Clock from synapse.util.frozenutils import freeze @@ -1079,3 +1080,139 @@ def test_delayed_message(self) -> None: # user2 should not be notified about it, because they can't see it. self.assertEqual(self.get_notif_count(self.user_id2), 0) + + +class PushRuleEvaluatorBaseRulesTestCase(unittest.TestCase): + def test_reactions(self) -> None: + message_event = FrozenEvent( + { + "event_id": "$event_id", + "room_id": "!room_id:beeper.com", + "content": { + "body": "Looks like Nick is way ahead of me on this one", + "msgtype": "m.text", + }, + "sender": "@brad:beeper.com", + "type": "m.room.message", + }, + RoomVersions.V1, + ) + + reaction_event = FrozenEvent( + { + "event_id": "$reaction_id", + "room_id": "!room_id:beeper.com", + "content": { + "m.relates_to": { + "event_id": "$event_id", + "key": "\U0001f44d", + "rel_type": "m.annotation", + } + }, + "sender": "@nick:beeper.com", + "type": "m.reaction", + }, + RoomVersions.V1, + ) + + dm_evaluator = PushRuleEvaluator( + _flatten_dict(reaction_event), + False, + 2, + 0, + {}, + {"m.annotation": _flatten_dict(message_event)}, + True, + reaction_event.room_version.msc3931_push_features, + True, + True, + False, + ) + + # Reaction to Brad's message, should be an action for Brad + actions = dm_evaluator.run( + FilteredPushRules(PushRules([]), {}, True, True, True, True, True, True), + "@brad:beeper.com", + "Brad", + False, + ) + self.assertTrue("notify" in actions) + + # Reaction to Brad's message, should not be an action for Nick + actions = dm_evaluator.run( + FilteredPushRules(PushRules([]), {}, True, True, True, True, True, True), + "@nick:beeper.com", + "Nick", + False, + ) + self.assertEqual(actions, []) + + large_room_evaluator = PushRuleEvaluator( + _flatten_dict(reaction_event), + False, + 30, + 0, + {}, + {"m.annotation": _flatten_dict(message_event)}, + True, + reaction_event.room_version.msc3931_push_features, + True, + True, + False, + ) + + # Large rooms should never have emoji reaction notifications + actions = large_room_evaluator.run( + FilteredPushRules(PushRules([]), {}, True, True, True, True, True, True), + "@brad:beeper.com", + "Brad", + False, + ) + self.assertEqual(actions, []) + actions = large_room_evaluator.run( + FilteredPushRules(PushRules([]), {}, True, True, True, True, True, True), + "@nick:beeper.com", + "Nick", + False, + ) + self.assertEqual(actions, []) + + def test_supress_auto_accept_invite(self) -> None: + event = FrozenEvent( + { + "event_id": "$event_id", + "room_id": "!wFyjEwanOaElpGOaLW:beeper.com", + "content": { + "displayname": "Brad Murray", + "fi.mau.will_auto_accept": True, + "is_direct": True, + "membership": "invite", + }, + "sender": "@_brad_imessagecloud_83372:beeper.com", + "state_key": "@brad:beeper.com", + "type": "m.room.member", + }, + RoomVersions.V1, + ) + + evaluator = PushRuleEvaluator( + _flatten_dict(event), + False, + 0, + 0, + {}, + {}, + True, + event.room_version.msc3931_push_features, + True, + True, + False, + ) + + actions = evaluator.run( + FilteredPushRules(PushRules([]), {}, True, True, True, True, True, True), + "@brad:beeper.com", + "Brad Murray", + False, + ) + self.assertEqual(actions, []) diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index b7b94482ef..4dd5c6dc9a 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -290,7 +290,7 @@ def build_event( self.master_store.add_push_actions_to_staging( event.event_id, dict(push_actions), - False, + {user_id: False for user_id, _ in push_actions}, "main", ) ) diff --git a/tests/rest/client/test_account_data.py b/tests/rest/client/test_account_data.py index be6d7af2fc..aed7f325c3 100644 --- a/tests/rest/client/test_account_data.py +++ b/tests/rest/client/test_account_data.py @@ -20,6 +20,7 @@ # from unittest.mock import AsyncMock +from synapse.api.constants import ReceiptTypes from synapse.rest import admin from synapse.rest.client import account_data, login, room @@ -79,3 +80,137 @@ def test_on_account_data_updated_callback(self) -> None: mocked_callback.assert_called_with( user_id, room_id, account_data_type, account_data_content ) + + def test_beeper_inbox_state_endpoint(self) -> None: + store = self.hs.get_datastores().main + + user_id = self.register_user("user", "password") + tok = self.login("user", "password") + + room_id = self.helper.create_room_as(user_id, tok=tok) + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/com.beeper.inbox/user/{user_id}/rooms/{room_id}/inbox_state", + {}, + access_token=tok, + ) + + self.assertEqual(channel.code, 200, channel.result) + self.assertIsNone( + self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, "com.beeper.inbox.done" + ) + ) + ) + self.assertIsNone( + self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, "m.marked_unread" + ) + ) + ) + + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/com.beeper.inbox/user/{user_id}/rooms/{room_id}/inbox_state", + {"done": {"at_delta": 1000 * 60 * 5}, "marked_unread": True}, + access_token=tok, + ) + + self.assertEqual(channel.code, 200, channel.result) + + # FIXME: I give up, I don't know how to mock time in tests + # ts = self.clock.time_msec() + ts = 500 + + done = self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, "com.beeper.inbox.done" + ) + ) + assert done is not None + self.assertEqual(done["updated_ts"], ts) + self.assertEqual(done["at_ts"], ts + (1000 * 60 * 5)) + + marked_unread = self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, "m.marked_unread" + ) + ) + assert marked_unread is not None + self.assertEqual(marked_unread["unread"], True) + self.assertEqual(marked_unread["ts"], ts) + + def test_beeper_inbox_state_endpoint_can_clear_unread(self) -> None: + store = self.hs.get_datastores().main + + user_id = self.register_user("user", "password") + tok = self.login("user", "password") + + room_id = self.helper.create_room_as(user_id, tok=tok) + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/com.beeper.inbox/user/{user_id}/rooms/{room_id}/inbox_state", + {"marked_unread": False}, + access_token=tok, + ) + + self.assertEqual(channel.code, 200, channel.result) + + # FIXME: I give up, I don't know how to mock time in tests + # ts = self.clock.time_msec() + ts = 400 + + self.assertEqual(channel.code, 200, channel.result) + self.assertIsNone( + self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, "com.beeper.inbox.done" + ) + ) + ) + + marked_unread = self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, "m.marked_unread" + ) + ) + assert marked_unread is not None + self.assertEqual(marked_unread["unread"], False) + self.assertEqual(marked_unread["ts"], ts) + + def test_beeper_inbox_state_endpoint_can_set_read_marker(self) -> None: + store = self.hs.get_datastores().main + + user_id = self.register_user("user", "password") + tok = self.login("user", "password") + + room_id = self.helper.create_room_as(user_id, tok=tok) + + res = self.helper.send(room_id, "hello", tok=tok) + + existing_read_marker = self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, ReceiptTypes.FULLY_READ + ) + ) + + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/com.beeper.inbox/user/{user_id}/rooms/{room_id}/inbox_state", + { + "read_markers": { + ReceiptTypes.FULLY_READ: res["event_id"], + }, + }, + access_token=tok, + ) + self.assertEqual(channel.code, 200) + + new_read_marker = self.get_success( + store.get_account_data_for_room_and_type( + user_id, room_id, ReceiptTypes.FULLY_READ + ) + ) + self.assertNotEqual(existing_read_marker, new_read_marker) diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 2d8ba77a77..dc314bcd96 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -20,7 +20,7 @@ # import urllib.parse -from typing import Any, Callable +from typing import Any from unittest.mock import AsyncMock, patch from twisted.internet.testing import MemoryReactor @@ -29,7 +29,6 @@ from synapse.rest import admin from synapse.rest.client import login, register, relations, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict from synapse.util.clock import Clock from tests import unittest @@ -136,29 +135,6 @@ def _get_related_events(self) -> list[str]: self.assertEqual(200, channel.code, channel.json_body) return [ev["event_id"] for ev in channel.json_body["chunk"]] - def _get_bundled_aggregations(self) -> JsonDict: - """ - Requests /event on the parent ID and returns the m.relations field (from unsigned), if it exists. - """ - # Fetch the bundled aggregations of the event. - channel = self.make_request( - "GET", - f"/_matrix/client/v3/rooms/{self.room}/event/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - return channel.json_body["unsigned"].get("m.relations", {}) - - def _find_event_in_chunk(self, events: list[JsonDict]) -> JsonDict: - """ - Find the parent event in a chunk of events and assert that it has the proper bundled aggregations. - """ - for event in events: - if event["event_id"] == self.parent_id: - return event - - raise AssertionError(f"Event {self.parent_id} not found in chunk") - class RelationsTestCase(BaseRelationsTestCase): def test_send_relation(self) -> None: @@ -361,321 +337,6 @@ def test_ignore_invalid_room(self) -> None: self.assertEqual(200, channel.code, channel.json_body) self.assertNotIn("m.relations", channel.json_body["unsigned"]) - def _assert_edit_bundle( - self, event_json: JsonDict, edit_event_id: str, edit_event_content: JsonDict - ) -> None: - """ - Assert that the given event has a correctly-serialised edit event in its - bundled aggregations - - Args: - event_json: the serialised event to be checked - edit_event_id: the ID of the edit event that we expect to be bundled - edit_event_content: the content of that event, excluding the 'm.relates_to` - property - """ - relations_dict = event_json["unsigned"].get("m.relations") - self.assertIn(RelationTypes.REPLACE, relations_dict) - - m_replace_dict = relations_dict[RelationTypes.REPLACE] - for key in [ - "event_id", - "sender", - "origin_server_ts", - "content", - "type", - "unsigned", - ]: - self.assertIn(key, m_replace_dict) - - expected_edit_content = { - "m.relates_to": { - "event_id": event_json["event_id"], - "rel_type": "m.replace", - } - } - expected_edit_content.update(edit_event_content) - - self.assert_dict( - { - "event_id": edit_event_id, - "sender": self.user_id, - "content": expected_edit_content, - "type": "m.room.message", - }, - m_replace_dict, - ) - - def test_edit(self) -> None: - """Test that a simple edit works.""" - orig_body = {"body": "Hi!", "msgtype": "m.text"} - new_body = {"msgtype": "m.text", "body": "I've been edited!"} - edit_event_content = { - "msgtype": "m.text", - "body": "foo", - "m.new_content": new_body, - } - channel = self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content=edit_event_content, - ) - edit_event_id = channel.json_body["event_id"] - - # /event should return the *original* event - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], orig_body) - self._assert_edit_bundle(channel.json_body, edit_event_id, edit_event_content) - - # Request the room messages. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/messages?dir=b", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self._assert_edit_bundle( - self._find_event_in_chunk(channel.json_body["chunk"]), - edit_event_id, - edit_event_content, - ) - - # Request the room context. - # /context should return the event. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/context/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self._assert_edit_bundle( - channel.json_body["event"], edit_event_id, edit_event_content - ) - self.assertEqual(channel.json_body["event"]["content"], orig_body) - - # Request sync, but limit the timeline so it becomes limited (and includes - # bundled aggregations). - filter = urllib.parse.quote_plus(b'{"room": {"timeline": {"limit": 2}}}') - channel = self.make_request( - "GET", f"/sync?filter={filter}", access_token=self.user_token - ) - self.assertEqual(200, channel.code, channel.json_body) - room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] - self.assertTrue(room_timeline["limited"]) - self._assert_edit_bundle( - self._find_event_in_chunk(room_timeline["events"]), - edit_event_id, - edit_event_content, - ) - - # Request search. - channel = self.make_request( - "POST", - "/search", - # Search term matches the parent message. - content={"search_categories": {"room_events": {"search_term": "Hi"}}}, - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - chunk = [ - result["result"] - for result in channel.json_body["search_categories"]["room_events"][ - "results" - ] - ] - self._assert_edit_bundle( - self._find_event_in_chunk(chunk), - edit_event_id, - edit_event_content, - ) - - def test_multi_edit(self) -> None: - """Test that multiple edits, including attempts by people who - shouldn't be allowed, are correctly handled. - """ - orig_body = orig_body = {"body": "Hi!", "msgtype": "m.text"} - self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content={ - "msgtype": "m.text", - "body": "Wibble", - "m.new_content": {"msgtype": "m.text", "body": "First edit"}, - }, - ) - - new_body = {"msgtype": "m.text", "body": "I've been edited!"} - edit_event_content = { - "msgtype": "m.text", - "body": "foo", - "m.new_content": new_body, - } - channel = self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content=edit_event_content, - ) - edit_event_id = channel.json_body["event_id"] - - self._send_relation( - RelationTypes.REPLACE, - "m.room.message.WRONG_TYPE", - content={ - "msgtype": "m.text", - "body": "Wibble", - "m.new_content": {"msgtype": "m.text", "body": "Edit, but wrong type"}, - }, - ) - - channel = self.make_request( - "GET", - f"/rooms/{self.room}/context/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - - self.assertEqual(channel.json_body["event"]["content"], orig_body) - self._assert_edit_bundle( - channel.json_body["event"], edit_event_id, edit_event_content - ) - - def test_edit_reply(self) -> None: - """Test that editing a reply works.""" - - # Create a reply to edit. - original_body = {"msgtype": "m.text", "body": "A reply!"} - channel = self._send_relation( - RelationTypes.REFERENCE, "m.room.message", content=original_body - ) - reply = channel.json_body["event_id"] - - edit_event_content = { - "msgtype": "m.text", - "body": "foo", - "m.new_content": {"msgtype": "m.text", "body": "I've been edited!"}, - } - channel = self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content=edit_event_content, - parent_id=reply, - ) - edit_event_id = channel.json_body["event_id"] - - # /event returns the original event - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{reply}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - event_result = channel.json_body - self.assertLessEqual(original_body.items(), event_result["content"].items()) - - # also check /context, which returns the *edited* event - channel = self.make_request( - "GET", - f"/rooms/{self.room}/context/{reply}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - context_result = channel.json_body["event"] - - # check that the relations are correct for both APIs - for result_event_dict, desc in ( - (event_result, "/event"), - (context_result, "/context"), - ): - # The reference metadata should still be intact. - self.assertLessEqual( - { - "m.relates_to": { - "event_id": self.parent_id, - "rel_type": "m.reference", - } - }.items(), - result_event_dict["content"].items(), - desc, - ) - - # We expect that the edit relation appears in the unsigned relations - # section. - self._assert_edit_bundle( - result_event_dict, edit_event_id, edit_event_content - ) - - def test_edit_edit(self) -> None: - """Test that an edit cannot be edited.""" - orig_body = {"body": "Hi!", "msgtype": "m.text"} - new_body = {"msgtype": "m.text", "body": "Initial edit"} - edit_event_content = { - "msgtype": "m.text", - "body": "Wibble", - "m.new_content": new_body, - } - channel = self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content=edit_event_content, - ) - edit_event_id = channel.json_body["event_id"] - - # Edit the edit event. - self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content={ - "msgtype": "m.text", - "body": "foo", - "m.new_content": {"msgtype": "m.text", "body": "Ignored edit"}, - }, - parent_id=edit_event_id, - ) - - # Request the original event. - # /event should return the original event. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], orig_body) - - # The relations information should not include the edit to the edit. - self._assert_edit_bundle(channel.json_body, edit_event_id, edit_event_content) - - # /context should return the bundled edit for the *first* edit - # (The edit to the edit should be ignored.) - channel = self.make_request( - "GET", - f"/rooms/{self.room}/context/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["event"]["content"], orig_body) - self._assert_edit_bundle( - channel.json_body["event"], edit_event_id, edit_event_content - ) - - # Directly requesting the edit should not have the edit to the edit applied. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{edit_event_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual("Wibble", channel.json_body["content"]["body"]) - self.assertIn("m.new_content", channel.json_body["content"]) - - # The relations information should not include the edit to the edit. - self.assertNotIn("m.relations", channel.json_body["unsigned"]) - def test_unknown_relations(self) -> None: """Unknown relations should be accepted.""" channel = self._send_relation("m.relation.test", "m.room.test") @@ -1072,774 +733,7 @@ def test_recursive_relations_with_filter(self) -> None: self.assertEqual(event_ids, [annotation_1]) -class BundledAggregationsTestCase(BaseRelationsTestCase): - """ - See RelationsTestCase.test_edit for a similar test for edits. - - Note that this doesn't test against /relations since only thread relations - get bundled via that API. See test_aggregation_get_event_for_thread. - """ - - def _test_bundled_aggregations( - self, - relation_type: str, - assertion_callable: Callable[[JsonDict], None], - expected_db_txn_for_event: int, - access_token: str | None = None, - ) -> None: - """ - Makes requests to various endpoints which should include bundled aggregations - and then calls an assertion function on the bundled aggregations. - - Args: - relation_type: The field to search for in the `m.relations` field in unsigned. - assertion_callable: Called with the contents of unsigned["m.relations"][relation_type] - for relation-specific assertions. - expected_db_txn_for_event: The number of database transactions which - are expected for a call to /event/. - access_token: The access token to user, defaults to self.user_token. - """ - access_token = access_token or self.user_token - - def assert_bundle(event_json: JsonDict) -> None: - """Assert the expected values of the bundled aggregations.""" - relations_dict = event_json["unsigned"].get("m.relations") - - # Ensure the fields are as expected. - self.assertCountEqual(relations_dict.keys(), (relation_type,)) - assertion_callable(relations_dict[relation_type]) - - # Request the event directly. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{self.parent_id}", - access_token=access_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - assert_bundle(channel.json_body) - assert channel.resource_usage is not None - self.assertEqual(channel.resource_usage.db_txn_count, expected_db_txn_for_event) - - # Request the room messages. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/messages?dir=b", - access_token=access_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - assert_bundle(self._find_event_in_chunk(channel.json_body["chunk"])) - - # Request the room context. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/context/{self.parent_id}", - access_token=access_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - assert_bundle(channel.json_body["event"]) - - # Request sync. - filter = urllib.parse.quote_plus(b'{"room": {"timeline": {"limit": 4}}}') - channel = self.make_request( - "GET", f"/sync?filter={filter}", access_token=access_token - ) - self.assertEqual(200, channel.code, channel.json_body) - room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] - self.assertTrue(room_timeline["limited"]) - assert_bundle(self._find_event_in_chunk(room_timeline["events"])) - - # Request search. - channel = self.make_request( - "POST", - "/search", - # Search term matches the parent message. - content={"search_categories": {"room_events": {"search_term": "Hi"}}}, - access_token=access_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - chunk = [ - result["result"] - for result in channel.json_body["search_categories"]["room_events"][ - "results" - ] - ] - assert_bundle(self._find_event_in_chunk(chunk)) - - def test_reference(self) -> None: - """ - Test that references get correctly bundled. - """ - channel = self._send_relation(RelationTypes.REFERENCE, "m.room.test") - reply_1 = channel.json_body["event_id"] - - channel = self._send_relation(RelationTypes.REFERENCE, "m.room.test") - reply_2 = channel.json_body["event_id"] - - def assert_annotations(bundled_aggregations: JsonDict) -> None: - self.assertEqual( - {"chunk": [{"event_id": reply_1}, {"event_id": reply_2}]}, - bundled_aggregations, - ) - - self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7) - - def test_thread(self) -> None: - """ - Test that threads get correctly bundled. - """ - # The root message is from "user", send replies as "user2". - self._send_relation( - RelationTypes.THREAD, "m.room.test", access_token=self.user2_token - ) - channel = self._send_relation( - RelationTypes.THREAD, "m.room.test", access_token=self.user2_token - ) - thread_2 = channel.json_body["event_id"] - - # This needs two assertion functions which are identical except for whether - # the current_user_participated flag is True, create a factory for the - # two versions. - def _gen_assert(participated: bool) -> Callable[[JsonDict], None]: - def assert_thread(bundled_aggregations: JsonDict) -> None: - self.assertEqual(2, bundled_aggregations.get("count")) - self.assertEqual( - participated, bundled_aggregations.get("current_user_participated") - ) - # The latest thread event has some fields that don't matter. - self.assertIn("latest_event", bundled_aggregations) - self.assert_dict( - { - "content": { - "m.relates_to": { - "event_id": self.parent_id, - "rel_type": RelationTypes.THREAD, - } - }, - "event_id": thread_2, - "sender": self.user2_id, - "type": "m.room.test", - }, - bundled_aggregations["latest_event"], - ) - - return assert_thread - - # The "user" sent the root event and is making queries for the bundled - # aggregations: they have participated. - self._test_bundled_aggregations(RelationTypes.THREAD, _gen_assert(True), 7) - # The "user2" sent replies in the thread and is making queries for the - # bundled aggregations: they have participated. - # - # Note that this re-uses some cached values, so the total number of - # queries is much smaller. - self._test_bundled_aggregations( - RelationTypes.THREAD, _gen_assert(True), 4, access_token=self.user2_token - ) - - # A user with no interactions with the thread: they have not participated. - user3_id, user3_token = self._create_user("charlie") - self.helper.join(self.room, user=user3_id, tok=user3_token) - self._test_bundled_aggregations( - RelationTypes.THREAD, _gen_assert(False), 4, access_token=user3_token - ) - - def test_thread_with_bundled_aggregations_for_latest(self) -> None: - """ - Bundled aggregations should get applied to the latest thread event. - """ - self._send_relation(RelationTypes.THREAD, "m.room.test") - channel = self._send_relation(RelationTypes.THREAD, "m.room.test") - thread_2 = channel.json_body["event_id"] - - channel = self._send_relation( - RelationTypes.REFERENCE, "org.matrix.test", parent_id=thread_2 - ) - reference_event_id = channel.json_body["event_id"] - - def assert_thread(bundled_aggregations: JsonDict) -> None: - self.assertEqual(2, bundled_aggregations.get("count")) - self.assertTrue(bundled_aggregations.get("current_user_participated")) - # The latest thread event has some fields that don't matter. - self.assertIn("latest_event", bundled_aggregations) - self.assert_dict( - { - "content": { - "m.relates_to": { - "event_id": self.parent_id, - "rel_type": RelationTypes.THREAD, - } - }, - "event_id": thread_2, - "sender": self.user_id, - "type": "m.room.test", - }, - bundled_aggregations["latest_event"], - ) - # Check the unsigned field on the latest event. - self.assert_dict( - { - "m.relations": { - RelationTypes.REFERENCE: { - "chunk": [{"event_id": reference_event_id}] - }, - } - }, - bundled_aggregations["latest_event"].get("unsigned"), - ) - - self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 7) - - def test_nested_thread(self) -> None: - """ - Ensure that a nested thread gets ignored by bundled aggregations, as - those are forbidden. - """ - - # Start a thread. - channel = self._send_relation(RelationTypes.THREAD, "m.room.test") - reply_event_id = channel.json_body["event_id"] - - # Disable the validation to pretend this came over federation, since it is - # not an event the Client-Server API will allow.. - with patch( - "synapse.handlers.message.EventCreationHandler._validate_event_relation", - new_callable=AsyncMock, - return_value=None, - ): - # Create a sub-thread off the thread, which is not allowed. - self._send_relation( - RelationTypes.THREAD, "m.room.test", parent_id=reply_event_id - ) - - # Fetch the thread root, to get the bundled aggregation for the thread. - relations_from_event = self._get_bundled_aggregations() - - # Ensure that requesting the room messages also does not return the sub-thread. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/messages?dir=b", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - event = self._find_event_in_chunk(channel.json_body["chunk"]) - relations_from_messages = event["unsigned"]["m.relations"] - - # Check the bundled aggregations from each point. - for aggregations, desc in ( - (relations_from_event, "/event"), - (relations_from_messages, "/messages"), - ): - # The latest event should have bundled aggregations. - self.assertIn(RelationTypes.THREAD, aggregations, desc) - thread_summary = aggregations[RelationTypes.THREAD] - self.assertIn("latest_event", thread_summary, desc) - self.assertEqual( - thread_summary["latest_event"]["event_id"], reply_event_id, desc - ) - - # The latest event should not have any bundled aggregations (since the - # only relation to it is another thread, which is invalid). - self.assertNotIn( - "m.relations", thread_summary["latest_event"]["unsigned"], desc - ) - - def test_thread_edit_latest_event(self) -> None: - """Test that editing the latest event in a thread works.""" - - # Create a thread and edit the last event. - channel = self._send_relation( - RelationTypes.THREAD, - "m.room.message", - content={"msgtype": "m.text", "body": "A threaded reply!"}, - ) - threaded_event_id = channel.json_body["event_id"] - - new_body = {"msgtype": "m.text", "body": "I've been edited!"} - channel = self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, - parent_id=threaded_event_id, - ) - edit_event_id = channel.json_body["event_id"] - - # Fetch the thread root, to get the bundled aggregation for the thread. - relations_dict = self._get_bundled_aggregations() - - # We expect that the edit message appears in the thread summary in the - # unsigned relations section. - self.assertIn(RelationTypes.THREAD, relations_dict) - - thread_summary = relations_dict[RelationTypes.THREAD] - self.assertIn("latest_event", thread_summary) - latest_event_in_thread = thread_summary["latest_event"] - # The latest event in the thread should have the edit appear under the - # bundled aggregations. - self.assertLessEqual( - {"event_id": edit_event_id, "sender": "@alice:test"}.items(), - latest_event_in_thread["unsigned"]["m.relations"][ - RelationTypes.REPLACE - ].items(), - ) - - def test_aggregation_get_event_for_annotation(self) -> None: - """Test that annotations do not get bundled aggregations included - when directly requested. - """ - channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction", "a") - annotation_id = channel.json_body["event_id"] - - # Annotate the annotation. - self._send_relation( - RelationTypes.ANNOTATION, "m.reaction", "a", parent_id=annotation_id - ) - - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{annotation_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertIsNone(channel.json_body["unsigned"].get("m.relations")) - - def test_aggregation_get_event_for_thread(self) -> None: - """Test that threads get bundled aggregations included when directly requested.""" - channel = self._send_relation(RelationTypes.THREAD, "m.room.test") - thread_id = channel.json_body["event_id"] - - # Make a reference to the thread. - channel = self._send_relation( - RelationTypes.REFERENCE, "org.matrix.test", parent_id=thread_id - ) - reference_event_id = channel.json_body["event_id"] - - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{thread_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual( - channel.json_body["unsigned"].get("m.relations"), - { - RelationTypes.REFERENCE: {"chunk": [{"event_id": reference_event_id}]}, - }, - ) - - # It should also be included when the entire thread is requested. - channel = self.make_request( - "GET", - f"/_matrix/client/v1/rooms/{self.room}/relations/{self.parent_id}?limit=1", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(len(channel.json_body["chunk"]), 1) - - thread_message = channel.json_body["chunk"][0] - self.assertEqual( - thread_message["unsigned"].get("m.relations"), - { - RelationTypes.REFERENCE: {"chunk": [{"event_id": reference_event_id}]}, - }, - ) - - def test_bundled_aggregations_with_filter(self) -> None: - """ - If "unsigned" is an omitted field (due to filtering), adding the bundled - aggregations should not break. - - Note that the spec allows for a server to return additional fields beyond - what is specified. - """ - channel = self._send_relation(RelationTypes.REFERENCE, "org.matrix.test") - reference_event_id = channel.json_body["event_id"] - - # Note that the sync filter does not include "unsigned" as a field. - filter = urllib.parse.quote_plus( - b'{"event_fields": ["content", "event_id"], "room": {"timeline": {"limit": 3}}}' - ) - channel = self.make_request( - "GET", f"/sync?filter={filter}", access_token=self.user_token - ) - self.assertEqual(200, channel.code, channel.json_body) - - # Ensure the timeline is limited, find the parent event. - room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] - self.assertTrue(room_timeline["limited"]) - parent_event = self._find_event_in_chunk(room_timeline["events"]) - - # Ensure there's bundled aggregations on it. - self.assertIn("unsigned", parent_event) - self.assertEqual( - parent_event["unsigned"].get("m.relations"), - { - RelationTypes.REFERENCE: {"chunk": [{"event_id": reference_event_id}]}, - }, - ) - - -class RelationIgnoredUserTestCase(BaseRelationsTestCase): - """Relations sent from an ignored user should be ignored.""" - - def _test_ignored_user( - self, - relation_type: str, - allowed_event_ids: list[str], - ignored_event_ids: list[str], - ) -> tuple[JsonDict, JsonDict]: - """ - Fetch the relations and ensure they're all there, then ignore user2, and - repeat. - - Returns: - A tuple of two JSON dictionaries, each are bundled aggregations, the - first is from before the user is ignored, and the second is after. - """ - # Get the relations. - event_ids = self._get_related_events() - self.assertCountEqual(event_ids, allowed_event_ids + ignored_event_ids) - - # And the bundled aggregations. - before_aggregations = self._get_bundled_aggregations() - self.assertIn(relation_type, before_aggregations) - - # Ignore user2 and re-do the requests. - self.get_success( - self.store.add_account_data_for_user( - self.user_id, - AccountDataTypes.IGNORED_USER_LIST, - {"ignored_users": {self.user2_id: {}}}, - ) - ) - - # Get the relations. - event_ids = self._get_related_events() - self.assertCountEqual(event_ids, allowed_event_ids) - - # And the bundled aggregations. - after_aggregations = self._get_bundled_aggregations() - self.assertIn(relation_type, after_aggregations) - - return before_aggregations[relation_type], after_aggregations[relation_type] - - def test_reference(self) -> None: - """Aggregations should exclude reference relations from ignored users""" - channel = self._send_relation(RelationTypes.REFERENCE, "m.room.test") - allowed_event_ids = [channel.json_body["event_id"]] - - channel = self._send_relation( - RelationTypes.REFERENCE, "m.room.test", access_token=self.user2_token - ) - ignored_event_ids = [channel.json_body["event_id"]] - - before_aggregations, after_aggregations = self._test_ignored_user( - RelationTypes.REFERENCE, allowed_event_ids, ignored_event_ids - ) - - self.assertCountEqual( - [e["event_id"] for e in before_aggregations["chunk"]], - allowed_event_ids + ignored_event_ids, - ) - - self.assertCountEqual( - [e["event_id"] for e in after_aggregations["chunk"]], allowed_event_ids - ) - - def test_thread(self) -> None: - """Aggregations should exclude thread releations from ignored users""" - channel = self._send_relation(RelationTypes.THREAD, "m.room.test") - allowed_event_ids = [channel.json_body["event_id"]] - - channel = self._send_relation( - RelationTypes.THREAD, "m.room.test", access_token=self.user2_token - ) - ignored_event_ids = [channel.json_body["event_id"]] - - before_aggregations, after_aggregations = self._test_ignored_user( - RelationTypes.THREAD, allowed_event_ids, ignored_event_ids - ) - - self.assertEqual(before_aggregations["count"], 2) - self.assertTrue(before_aggregations["current_user_participated"]) - # The latest thread event has some fields that don't matter. - self.assertEqual( - before_aggregations["latest_event"]["event_id"], ignored_event_ids[0] - ) - - self.assertEqual(after_aggregations["count"], 1) - self.assertTrue(after_aggregations["current_user_participated"]) - # The latest thread event has some fields that don't matter. - self.assertEqual( - after_aggregations["latest_event"]["event_id"], allowed_event_ids[0] - ) - - -class RelationRedactionTestCase(BaseRelationsTestCase): - """ - Test the behaviour of relations when the parent or child event is redacted. - - The behaviour of each relation type is subtly different which causes the tests - to be a bit repetitive, they follow a naming scheme of: - - test_redact_(relation|parent)_{relation_type} - - The first bit of "relation" means that the event with the relation defined - on it (the child event) is to be redacted. A "parent" means that the target - of the relation (the parent event) is to be redacted. - - The relation_type describes which type of relation is under test (i.e. it is - related to the value of rel_type in the event content). - """ - - def _redact(self, event_id: str) -> None: - channel = self.make_request( - "POST", - f"/_matrix/client/r0/rooms/{self.room}/redact/{event_id}", - access_token=self.user_token, - content={}, - ) - self.assertEqual(200, channel.code, channel.json_body) - - def _get_threads(self) -> list[tuple[str, str]]: - """Request the threads in the room and returns a list of thread ID and latest event ID.""" - # Request the threads in the room. - channel = self.make_request( - "GET", - f"/_matrix/client/v1/rooms/{self.room}/threads", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - threads = channel.json_body["chunk"] - return [ - ( - t["event_id"], - t["unsigned"]["m.relations"][RelationTypes.THREAD]["latest_event"][ - "event_id" - ], - ) - for t in threads - ] - - def test_redact_relation_thread(self) -> None: - """ - Test that thread replies are properly handled after the thread reply redacted. - - The redacted event should not be included in bundled aggregations or - the response to relations. - """ - # Create a thread with a few events in it. - thread_replies = [] - for i in range(3): - channel = self._send_relation( - RelationTypes.THREAD, - EventTypes.Message, - content={"body": f"reply {i}", "msgtype": "m.text"}, - ) - thread_replies.append(channel.json_body["event_id"]) - - ################################################## - # Check the test data is configured as expected. # - ################################################## - self.assertEqual(self._get_related_events(), list(reversed(thread_replies))) - relations = self._get_bundled_aggregations() - self.assertLessEqual( - {"count": 3, "current_user_participated": True}.items(), - relations[RelationTypes.THREAD].items(), - ) - # The latest event is the last sent event. - self.assertEqual( - relations[RelationTypes.THREAD]["latest_event"]["event_id"], - thread_replies[-1], - ) - - # There should be one thread, the latest event is the event that will be redacted. - self.assertEqual(self._get_threads(), [(self.parent_id, thread_replies[-1])]) - - ########################## - # Redact the last event. # - ########################## - self._redact(thread_replies.pop()) - - # The thread should still exist, but the latest event should be updated. - self.assertEqual(self._get_related_events(), list(reversed(thread_replies))) - relations = self._get_bundled_aggregations() - self.assertLessEqual( - {"count": 2, "current_user_participated": True}.items(), - relations[RelationTypes.THREAD].items(), - ) - # And the latest event is the last unredacted event. - self.assertEqual( - relations[RelationTypes.THREAD]["latest_event"]["event_id"], - thread_replies[-1], - ) - self.assertEqual(self._get_threads(), [(self.parent_id, thread_replies[-1])]) - - ########################################### - # Redact the *first* event in the thread. # - ########################################### - self._redact(thread_replies.pop(0)) - - # Nothing should have changed (except the thread count). - self.assertEqual(self._get_related_events(), thread_replies) - relations = self._get_bundled_aggregations() - self.assertLessEqual( - {"count": 1, "current_user_participated": True}.items(), - relations[RelationTypes.THREAD].items(), - ) - # And the latest event is the last unredacted event. - self.assertEqual( - relations[RelationTypes.THREAD]["latest_event"]["event_id"], - thread_replies[-1], - ) - self.assertEqual(self._get_threads(), [(self.parent_id, thread_replies[-1])]) - - #################################### - # Redact the last remaining event. # - #################################### - self._redact(thread_replies.pop(0)) - self.assertEqual(thread_replies, []) - - # The event should no longer be considered a thread. - self.assertEqual(self._get_related_events(), []) - self.assertEqual(self._get_bundled_aggregations(), {}) - self.assertEqual(self._get_threads(), []) - - def test_redact_parent_edit(self) -> None: - """Test that edits of an event are redacted when the original event - is redacted. - """ - # Add a relation - self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - parent_id=self.parent_id, - content={ - "msgtype": "m.text", - "body": "Wibble", - "m.new_content": {"msgtype": "m.text", "body": "First edit"}, - }, - ) - - # Check the relation is returned - event_ids = self._get_related_events() - relations = self._get_bundled_aggregations() - self.assertEqual(len(event_ids), 1) - self.assertIn(RelationTypes.REPLACE, relations) - - # Redact the original event - self._redact(self.parent_id) - - # The relations are not returned. - event_ids = self._get_related_events() - relations = self._get_bundled_aggregations() - self.assertEqual(len(event_ids), 0) - self.assertEqual(relations, {}) - - def test_redact_parent_annotation(self) -> None: - """Test that annotations of an event are viewable when the original event - is redacted. - """ - # Add a relation - channel = self._send_relation(RelationTypes.REFERENCE, "org.matrix.test") - related_event_id = channel.json_body["event_id"] - - # The relations should exist. - event_ids = self._get_related_events() - relations = self._get_bundled_aggregations() - self.assertEqual(len(event_ids), 1) - self.assertIn(RelationTypes.REFERENCE, relations) - - # Redact the original event. - self._redact(self.parent_id) - - # The relations are returned. - event_ids = self._get_related_events() - relations = self._get_bundled_aggregations() - self.assertEqual(event_ids, [related_event_id]) - self.assertEqual( - relations[RelationTypes.REFERENCE], - {"chunk": [{"event_id": related_event_id}]}, - ) - - def test_redact_parent_thread(self) -> None: - """ - Test that thread replies are still available when the root event is redacted. - """ - channel = self._send_relation( - RelationTypes.THREAD, - EventTypes.Message, - content={"body": "reply 1", "msgtype": "m.text"}, - ) - related_event_id = channel.json_body["event_id"] - - # Redact one of the reactions. - self._redact(self.parent_id) - - # The unredacted relation should still exist. - event_ids = self._get_related_events() - relations = self._get_bundled_aggregations() - self.assertEqual(len(event_ids), 1) - self.assertLessEqual( - { - "count": 1, - "current_user_participated": True, - }.items(), - relations[RelationTypes.THREAD].items(), - ) - self.assertEqual( - relations[RelationTypes.THREAD]["latest_event"]["event_id"], - related_event_id, - ) - - class ThreadsTestCase(BaseRelationsTestCase): - def _get_threads(self, body: JsonDict) -> list[tuple[str, str]]: - return [ - ( - ev["event_id"], - ev["unsigned"]["m.relations"]["m.thread"]["latest_event"]["event_id"], - ) - for ev in body["chunk"] - ] - - def test_threads(self) -> None: - """Create threads and ensure the ordering is due to their latest event.""" - # Create 2 threads. - thread_1 = self.parent_id - res = self.helper.send(self.room, body="Thread Root!", tok=self.user_token) - thread_2 = res["event_id"] - - channel = self._send_relation(RelationTypes.THREAD, "m.room.test") - reply_1 = channel.json_body["event_id"] - channel = self._send_relation( - RelationTypes.THREAD, "m.room.test", parent_id=thread_2 - ) - reply_2 = channel.json_body["event_id"] - - # Request the threads in the room. - channel = self.make_request( - "GET", - f"/_matrix/client/v1/rooms/{self.room}/threads", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - threads = self._get_threads(channel.json_body) - self.assertEqual(threads, [(thread_2, reply_2), (thread_1, reply_1)]) - - # Update the first thread, the ordering should swap. - channel = self._send_relation(RelationTypes.THREAD, "m.room.test") - reply_3 = channel.json_body["event_id"] - - channel = self.make_request( - "GET", - f"/_matrix/client/v1/rooms/{self.room}/threads", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - # Tuple of (thread ID, latest event ID) for each thread. - threads = self._get_threads(channel.json_body) - self.assertEqual(threads, [(thread_1, reply_3), (thread_2, reply_2)]) - def test_pagination(self) -> None: """Create threads and paginate through them.""" # Create 2 threads. diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 74a8678ae9..140ea01be5 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -33,7 +33,16 @@ ReceiptTypes, RelationTypes, ) -from synapse.rest.client import devices, knock, login, read_marker, receipts, room, sync +from synapse.rest.client import ( + account_data, + devices, + knock, + login, + read_marker, + receipts, + room, + sync, +) from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util.clock import Clock @@ -460,6 +469,7 @@ def test_create_event_present_in_knock_state(self) -> None: class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ + account_data.register_servlets, synapse.rest.admin.register_servlets, login.register_servlets, read_marker.register_servlets, @@ -561,6 +571,9 @@ def test_unread_counts(self) -> None: # Check that the unread counter is back to 0. self._check_unread_count(0) + # Beeper: we don't count name as unread, so send this to increase the counter + self.helper.send_event(self.room_id, EventTypes.Encrypted, {}, tok=self.tok2) + # Check that room name changes increase the unread counter. self.helper.send_state( self.room_id, @@ -570,6 +583,9 @@ def test_unread_counts(self) -> None: ) self._check_unread_count(1) + # Beeper: we don't count topic as unread, so send this to increase the counter + self.helper.send_event(self.room_id, EventTypes.Encrypted, {}, tok=self.tok2) + # Check that room topic changes increase the unread counter. self.helper.send_state( self.room_id, @@ -583,6 +599,10 @@ def test_unread_counts(self) -> None: self.helper.send_event(self.room_id, EventTypes.Encrypted, {}, tok=self.tok2) self._check_unread_count(3) + # Beeper: fake event to bump event count, we don't count custom events + # as unread currently. + self.helper.send_event(self.room_id, EventTypes.Encrypted, {}, tok=self.tok2) + # Check that custom events with a body increase the unread counter. result = self.helper.send_event( self.room_id, @@ -616,7 +636,7 @@ def test_unread_counts(self) -> None: content={"body": "hello", "msgtype": "m.notice"}, tok=self.tok2, ) - self._check_unread_count(4) + self._check_unread_count(5) # Beep: notices count as unread # Check that tombstone events changes increase the unread counter. res1 = self.helper.send_state( @@ -647,6 +667,28 @@ def test_unread_counts(self) -> None: self.assertEqual(channel.code, 200, channel.json_body) self._check_unread_count(0) + def test_beeper_inbox_state_can_update_unread_count(self) -> None: + # increase unread count + self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + res = self.helper.send(self.room_id, "hello", tok=self.tok2) + self._check_unread_count(1) + + # Beeper: inbox_state should be able to send read receipts + res = self.helper.send(self.room_id, "hello", tok=self.tok2) + + channel = self.make_request( + "PUT", + f"/_matrix/client/unstable/com.beeper.inbox/user/{self.user_id}/rooms/{self.room_id}/inbox_state", + { + "read_markers": { + ReceiptTypes.READ: res["event_id"], + }, + }, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self._check_unread_count(0) + # We test for all three receipt types that influence notification counts @parameterized.expand( [ @@ -1272,3 +1314,348 @@ def test_incremental_sync(self) -> None: ) self.assertEqual(200, channel.code, msg=channel.result["body"]) + + +class BeeperRoomPreviewTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + read_marker.register_servlets, + room.register_servlets, + sync.register_servlets, + receipts.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.url = "/sync?beeper_previews=true&since=%s" + self.next_batches = {} + + # Register the first user (used to check the unread counts). + self.user_id = self.register_user("kermit", "monkey") + self.tok = self.login("kermit", "monkey") + self.next_batches[self.tok] = "s0" + + # Create the room we'll check unread counts for. + self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok) + self.room_id_2 = self.helper.create_room_as(self.user_id, tok=self.tok) + self.room_id_3 = self.helper.create_room_as(self.user_id, tok=self.tok) + self.room_id_4 = self.helper.create_room_as(self.user_id, tok=self.tok) + + # Register the second user (used to send events to the room). + self.user2 = self.register_user("kermit2", "monkey") + self.tok2 = self.login("kermit2", "monkey") + self.next_batches[self.tok2] = "s0" + + # Change the power levels of the room so that the second user can send state + # events. + self.helper.send_state( + self.room_id, + EventTypes.PowerLevels, + { + "users": {self.user_id: 100, self.user2: 100}, + "users_default": 0, + "events": { + "m.room.name": 50, + "m.room.power_levels": 100, + "m.room.history_visibility": 100, + "m.room.canonical_alias": 50, + "m.room.avatar": 50, + "m.room.tombstone": 100, + "m.room.server_acl": 100, + "m.room.encryption": 100, + }, + "events_default": 0, + "state_default": 50, + "ban": 50, + "kick": 50, + "redact": 50, + "invite": 0, + }, + tok=self.tok, + ) + + def _check_preview_event_ids(self, auth_token: str, expected: dict) -> None: + """Checks the populated preview value against the expected value provided""" + + channel = self.make_request( + "GET", + self.url % self.next_batches[auth_token], + access_token=auth_token, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + + for room_id, expected_entry in expected.items(): + room_entry = ( + channel.json_body.get("rooms", {}).get("join", {}).get(room_id, {}) + ) + + preview = room_entry.get("com.beeper.inbox.preview") + if preview: + preview_id = preview.get("event_id") + self.assertEqual( + preview_id, + expected_entry, + room_entry, + ) + else: + self.assertIsNone(expected_entry, room_entry) + + # Store the next batch for the next request. + self.next_batches[auth_token] = channel.json_body["next_batch"] + + def _redact_event( + self, + access_token: str, + room_id: str, + event_id: str, + expect_code: int = 200, + with_relations: list[str] | None = None, + ) -> JsonDict: + """Helper function to send a redaction event. + + Returns the json body. + """ + path = "/_matrix/client/r0/rooms/%s/redact/%s" % (room_id, event_id) + + request_content = {} + if with_relations: + request_content["org.matrix.msc3912.with_relations"] = with_relations + + channel = self.make_request( + "POST", path, request_content, access_token=access_token + ) + self.assertEqual(channel.code, expect_code) + return channel.json_body + + def test_room_previews(self) -> None: + """Tests that /sync returns all room previews on first sync.""" + + # Multiple events in rooms for first sync. + self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + self.helper.join(room=self.room_id_2, user=self.user2, tok=self.tok2) + self.helper.join(room=self.room_id_3, user=self.user2, tok=self.tok2) + self.helper.join(room=self.room_id_4, user=self.user2, tok=self.tok2) + + send_body = self.helper.send(self.room_id, "hello", tok=self.tok2) + send_body2 = self.helper.send(self.room_id_2, "hello 2", tok=self.tok2) + send_body3 = self.helper.send(self.room_id_3, "hello 3", tok=self.tok2) + send_body4 = self.helper.send(self.room_id_4, "hello 4", tok=self.tok2) + + # Should have previews for all rooms on first sync. + self._check_preview_event_ids( + auth_token=self.tok, + expected={ + self.room_id: send_body["event_id"], + self.room_id_2: send_body2["event_id"], + self.room_id_3: send_body3["event_id"], + self.room_id_4: send_body4["event_id"], + }, + ) + + # Subsequent - update preview for only room 2" + send_body5 = self.helper.send(self.room_id_2, "Sup!", tok=self.tok2) + + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id_2: send_body5["event_id"]} + ) + + def test_room_preview(self) -> None: + """Tests that /sync returns a room preview with the latest message for room.""" + + # One user says hello. + # Check that a message we send returns a preview in the room (i.e. have multiple clients?) + send_body = self.helper.send(self.room_id, "hello", tok=self.tok) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: send_body["event_id"]} + ) + + # Join new user. Should not show updated preview. + self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: send_body["event_id"]} + ) + + # Second user says hello + # Check that the new user sending a message updates our preview + send_2_body = self.helper.send(self.room_id, "hello again!", tok=self.tok2) + self._check_preview_event_ids(self.tok, {self.room_id: send_2_body["event_id"]}) + + # Encrypted messages 1 + # Beeper: ensure encrypted messages are treated the same. + enc_1_body = self.helper.send_event( + self.room_id, EventTypes.Encrypted, {}, tok=self.tok2 + ) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: enc_1_body["event_id"]} + ) + + # Encrypted messages 2 + enc_2_body = self.helper.send_event( + self.room_id, EventTypes.Encrypted, {}, tok=self.tok2 + ) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: enc_2_body["event_id"]} + ) + + # Redact encrypted message 2 + self._redact_event(self.tok2, self.room_id, enc_2_body["event_id"]) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: enc_1_body["event_id"]} + ) + + # User 2 react to user 1 message + # Someone else reacted to my message, update preview. + reaction_1 = self.helper.send_event( + room_id=self.room_id, + type=EventTypes.Reaction, + content={ + "m.relates_to": { + "rel_type": RelationTypes.ANNOTATION, + "event_id": send_body["event_id"], + "key": "👍", + } + }, + tok=self.tok2, + ) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: reaction_1["event_id"]} + ) + + # User 1 react to User 2 message. + # Not a reaction to my message, don't update preview. + reaction_2 = self.helper.send_event( + room_id=self.room_id, + type=EventTypes.Reaction, + content={ + "m.relates_to": { + "rel_type": RelationTypes.ANNOTATION, + "event_id": send_2_body["event_id"], + "key": "👍", + } + }, + tok=self.tok, + ) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: reaction_1["event_id"]} + ) + self._check_preview_event_ids( + auth_token=self.tok2, expected={self.room_id: reaction_2["event_id"]} + ) + + # Redact user 2 message with reactions. + # Remove redactions as well as reactions from user 2's preview. + self._redact_event(self.tok2, self.room_id, send_2_body["event_id"]) + + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: reaction_1["event_id"]} + ) + self._check_preview_event_ids( + auth_token=self.tok2, expected={self.room_id: enc_1_body["event_id"]} + ) + + def test_room_preview_edits(self) -> None: + """Tests that /sync returns a room preview with the latest message for room.""" + + # One user says hello. + # Check that a message we send returns a preview in the room (i.e. have multiple clients?) + send_body = self.helper.send(self.room_id, "hello", tok=self.tok) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: send_body["event_id"]} + ) + + # Join new user. Should not show updated preview. + self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: send_body["event_id"]} + ) + + # Second user says hello + # Check that the new user sending a message updates our preview + send_2_body = self.helper.send(self.room_id, "hello again!", tok=self.tok2) + self._check_preview_event_ids(self.tok, {self.room_id: send_2_body["event_id"]}) + + # First user edits their old message + # Check that this doesn't alter the preview + self.helper.send_event( + room_id=self.room_id, + type=EventTypes.Message, + content={ + "body": "hello edit", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": RelationTypes.REPLACE, + "event_id": send_body["event_id"], + }, + }, + tok=self.tok, + ) + self._check_preview_event_ids(self.tok, {self.room_id: send_2_body["event_id"]}) + + # Now second user edits their (currently preview) message + # Check that this does become the preview + send_3_body = self.helper.send_event( + room_id=self.room_id, + type=EventTypes.Message, + content={ + "body": "hello edit", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": RelationTypes.REPLACE, + "event_id": send_2_body["event_id"], + }, + }, + tok=self.tok2, + ) + self._check_preview_event_ids(self.tok, {self.room_id: send_3_body["event_id"]}) + + # Now second user edits their (currently preview) message again + # Check that this does become the preview, over the previous edit + send_4_body = self.helper.send_event( + room_id=self.room_id, + type=EventTypes.Message, + content={ + "body": "hello edit 2", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": RelationTypes.REPLACE, + "event_id": send_2_body["event_id"], + }, + }, + tok=self.tok2, + ) + self._check_preview_event_ids(self.tok, {self.room_id: send_4_body["event_id"]}) + + # Finally, first user sends a message and this should become the preview + send_5_body = self.helper.send(self.room_id, "hello", tok=self.tok) + self._check_preview_event_ids( + auth_token=self.tok, expected={self.room_id: send_5_body["event_id"]} + ) + + def test_room_preview_no_change(self) -> None: + """Tests that /sync only includes previews when we have new events.""" + + self.helper.join(room=self.room_id, user=self.user_id, tok=self.tok) + + send_body = self.helper.send(self.room_id, "hello", tok=self.tok) + + # Should have preview on first sync + self._check_preview_event_ids( + auth_token=self.tok, + expected={self.room_id: send_body["event_id"]}, + ) + + # Should have no preview on second sync (no timeline changes) + self._check_preview_event_ids( + auth_token=self.tok, + expected={self.room_id: None}, + ) + + # Send a join event, this isn't previewed but will be in the timeline + self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + + # Should have preview because we have timeline, but preview is unchanged + self._check_preview_event_ids( + auth_token=self.tok, + expected={self.room_id: send_body["event_id"]}, + ) diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index 6cb85c94c4..a5127ea5b0 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -26,7 +26,7 @@ from synapse.api.room_versions import RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.rest import admin -from synapse.rest.client import login, room, room_upgrade_rest_servlet +from synapse.rest.client import login, notifications, room, room_upgrade_rest_servlet from synapse.server import HomeServer from synapse.util.clock import Clock @@ -40,6 +40,7 @@ class UpgradeRoomTest(unittest.HomeserverTestCase): login.register_servlets, room.register_servlets, room_upgrade_rest_servlet.register_servlets, + notifications.register_servlets, ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: @@ -481,3 +482,37 @@ def test_creator_removed_from_powerlevels_v12(self) -> None: # This is a regression test where previously Synapse would accidentally # mutate the old power levels event. self.assertEqual(old_power_level_event.content["users"][self.creator], 100) + + def test_upgrade_clears_push_actions(self) -> None: + """ + Beeper specific test: ensure that when upgrading a room any notification/unread counts + in the old room are removed. + """ + self.helper.send_event( + self.room_id, + "m.room.message", + content={"body": "hi", "msgtype": "text"}, + tok=self.other_token, + ) + + # Check we have a notification pre-upgrade + channel = self.make_request( + "GET", + "/notifications", + access_token=self.creator_token, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(len(channel.json_body["notifications"]), 1, channel.json_body) + + channel = self._upgrade_room() + self.assertEqual(200, channel.code, channel.result) + self.assertIn("replacement_room", channel.json_body) + + # Check we have no notification pre-upgrade + channel = self.make_request( + "GET", + "/notifications", + access_token=self.creator_token, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(len(channel.json_body["notifications"]), 0, channel.json_body) diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py index c73717f014..14a5f4eba3 100644 --- a/tests/rest/test_well_known.py +++ b/tests/rest/test_well_known.py @@ -146,6 +146,8 @@ def test_client_well_known_msc3861_oauth_delegation(self) -> None: "issuer": "https://issuer", "account": "https://my-account.issuer", }, + # Beep: added because iOS crashed without + "m.identity_server": {"base_url": ""}, }, ) diff --git a/tests/server.py b/tests/server.py index 20fcc42081..9e3b942ec4 100644 --- a/tests/server.py +++ b/tests/server.py @@ -102,6 +102,7 @@ from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock from synapse.util.json import json_encoder +from synapse.util.task_scheduler import TaskScheduler from tests.utils import ( LEAVE_DB, @@ -1107,6 +1108,10 @@ def setup_test_homeserver( if reactor is None: reactor = ThreadedMemoryReactorClock() + # Beep: the post-task sleep confuses tests that advance the fake reactor + # just far enough for a task to complete, so disable it. + TaskScheduler.SLEEP_AFTER_TASK_S = 0 + if config is None: config = default_config(server_name, parse=True) diff --git a/tests/storage/test_account_data.py b/tests/storage/test_account_data.py index c91aad097d..72549406ae 100644 --- a/tests/storage/test_account_data.py +++ b/tests/storage/test_account_data.py @@ -108,6 +108,22 @@ def test_ignoring_self_fails(self) -> None: self.assertEqual(f.code, 400) self.assertEqual(f.errcode, Codes.INVALID_PARAM) + def test_ignoring_bot_users(self) -> None: + self._update_ignore_list("@other:test", "@another:remote") + self.assert_ignored(self.user, {"@other:test", "@another:remote"}) + + self._update_ignore_list("@other:test", "@another:remote", "@_other_bot:test") + self.assert_ignored(self.user, {"@other:test", "@another:remote"}) + + self._update_ignore_list("@iamnotabot:beeper.com") + self.assert_ignored(self.user, {"@iamnotabot:beeper.com"}) + + self._update_ignore_list("@_other_bot:beeper.com") + self.assert_ignored(self.user, set()) + + self._update_ignore_list("@whatsappbot:beeper.local") + self.assert_ignored(self.user, set()) + def test_caching(self) -> None: """Ensure that caching works properly between different users.""" # The first user ignores a user. diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index bd68f2aaa1..8d07bf1a45 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -100,7 +100,7 @@ def test_insert_new_client_ip_none_device_id(self) -> None: user_id, "access_token", "ip", "user_agent", None ) ) - self.reactor.advance(200) + self.reactor.advance(3600) self.pump(0) result = cast( @@ -153,7 +153,7 @@ def test_insert_new_client_ip_none_device_id(self) -> None: ) # Only one result, has been upserted. self.assertEqual( - result, [("access_token", "ip", "user_agent", None, 12345878000)] + result, [("access_token", "ip", "user_agent", None, 12349278000)] ) @parameterized.expand([(False,), (True,)])