Bump version to v3.5.7

* Add a limit to how many posts can get fetched as a result of a single request

* Add tests

* Always pass `request_id` when processing `Announce` activities

---------

Co-authored-by: nametoolong <nametoolong@users.noreply.github.com>

.circleci/config.yml

@@ -68,7 +68,9 @@ jobs:
           cache-version: v1
           pkg-manager: yarn
       - run:
-          command: ./bin/rails assets:precompile
+          command: |
+            export NODE_OPTIONS=--openssl-legacy-provider
+            ./bin/rails assets:precompile
           name: Precompile assets
       - persist_to_workspace:
           paths:

.github/workflows/build-image.yml

@@ -10,33 +10,55 @@ on:
     paths:
       - .github/workflows/build-image.yml
       - Dockerfile
+
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build-image:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+      - uses: actions/checkout@v3
+      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: github.event_name != 'pull_request'
-      - uses: docker/metadata-action@v3
+        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
+
+      - name: Log in to the Github Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
+
+      - uses: docker/metadata-action@v4
         id: meta
         with:
-          images: tootsuite/mastodon
+          images: |
+            tootsuite/mastodon
+            ghcr.io/mastodon/mastodon
           flavor: |
             latest=auto
           tags: |
             type=edge,branch=main
             type=match,pattern=v(.*),group=0
             type=ref,event=pr
-      - uses: docker/build-push-action@v2
+
+      - uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
+          provenance: false
+          builder: ${{ steps.buildx.outputs.name }}
+          push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
-          cache-from: type=registry,ref=tootsuite/mastodon:latest
-          cache-to: type=inline
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

CHANGELOG.md

@@ -3,6 +3,74 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+# [3.5.7] - 2023-03-16
+
+### Added
+
+- Add `lang` attribute to native language names in language picker in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23749))
+- Add headers to outgoing mails to avoid auto-replies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23597))
+
+### Fixed
+
+- Fix “Remove all followers from the selected domains” being more destructive than it claims ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23805))
+- Fix case-sensitive check for previously used hashtags in hashtag autocompletion ([deanveloper](https://github.com/mastodon/mastodon/pull/23526))
+- Fix inefficiency when searching accounts per username in admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23801))
+- Fix server error when failing to follow back followers from `/relationships` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23787))
+- Fix original account being unfollowed on migration before the follow request to the new account could be sent ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21957))
+- Fix pgBouncer resetting application name on every transaction ([Gargron](https://github.com/mastodon/mastodon/pull/23958))
+- Fix unconfirmed accounts being counted as active users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23803))
+- Fix `/api/v1/streaming` sub-paths not being redirected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23988))
+- Fix drag'n'drop upload area text that spans multiple lines not being centered ([vintprox](https://github.com/mastodon/mastodon/pull/24029))
+- Fix sidekiq jobs not triggering Elasticsearch index updates ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24046))
+- Fix missing null check on applications on strike disputes ([kescherCode](https://github.com/mastodon/mastodon/pull/19851))
+- Fix dashboard crash on ElasticSearch server error ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23751))
+- Fix incorrect post links in strikes when the account is remote ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23611))
+- Fix misleading error code when receiving invalid WebAuthn credentials ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23568))
+
+### Security
+
+- Change user backups to use expiring URLs for download when possible ([Gargron](https://github.com/mastodon/mastodon/pull/24136))
+- Add warning for object storage misconfiguration ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24137))
+
+## [3.5.6] - 2023-02-09
+### Fixed
+
+- **Fix changing domain block severity not undoing individual account effects** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23480))
+- Fix suspension worker crashing on S3-compatible setups without ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23481))
+- Fix possible race conditions when suspending/unsuspending accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23482))
+- Fix some performance issues with `/admin/instances` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23483))
+- Fix voter count not being cleared when a poll is reset ([afontenot](https://github.com/mastodon/mastodon/pull/23484))
+- Fix attachments of edited statuses not being fetched ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23485))
+- Fix 500 error when marking posts as sensitive while some of them are deleted ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23486))
+- Fix user clean-up scheduler crash when an unconfirmed account has a moderation note ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23487))
+- Fix pending account approval and rejection not being recorded in the admin audit log ([FrancisMurillo](https://github.com/mastodon/mastodon/pull/23488))
+- Fix replies sometimes being delivered to user-blocked domains ([tribela](https://github.com/mastodon/mastodon/pull/23490))
+- Fix sanitizer parsing link text as HTML when stripping unsupported links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23491))
+- Fix REST API serializer for `Account` not including `moved` when the moved account has itself moved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23492))
+
+### Security
+
+- Add `form-action` CSP directive ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23478))
+- Fix unbounded recursion in account discovery ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/22026))
+- Fix unbounded recursion in post discovery ([ClearlyClaire,nametoolong](https://github.com/mastodon/mastodon/pull/23507))
+
+## [3.5.5] - 2022-11-14
+## Fixed
+
+- Fix nodes order being sometimes mangled when rewriting emoji ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20677))
+
+## [3.5.4] - 2022-11-14
+### Fixed
+
+- Fix error when a remote report includes a private post the server has no access to ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18760))
+
+### Security
+
+- Fix emoji substitution not applying only to text nodes in backend code ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20641))
+- Fix emoji substitution not applying only to text nodes in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20640))
+- Fix rate limiting for paths with formats ([Gargron](https://github.com/mastodon/mastodon/pull/20675))
+- Fix out-of-bound reads in blurhash transcoder ([delroth](https://github.com/mastodon/mastodon/pull/20388))
+
 ## [3.5.3] - 2022-05-26
 ### Added
 

Gemfile

@@ -66,6 +66,7 @@ gem 'oj', '~> 3.13'
 gem 'ox', '~> 2.14'
 gem 'parslet'
 gem 'posix-spawn'
+gem 'public_suffix', '~> 4.0.7'
 gem 'pundit', '~> 2.2'
 gem 'premailer-rails'
 gem 'rack-attack', '~> 6.6'

Gemfile.lock

@@ -803,6 +803,7 @@ DEPENDENCIES
   private_address_check (~> 0.5)
   pry-byebug (~> 3.9)
   pry-rails (~> 0.3)
+  public_suffix (~> 4.0.7)
   puma (~> 5.6)
   pundit (~> 2.2)
   rack (~> 2.2.3)

README.md

@@ -5,13 +5,11 @@
 [![Build Status](https://img.shields.io/circleci/project/github/mastodon/mastodon.svg)][circleci]
 [![Code Climate](https://img.shields.io/codeclimate/maintainability/mastodon/mastodon.svg)][code_climate]
 [![Crowdin](https://d322cqt584bo4o.cloudfront.net/mastodon/localized.svg)][crowdin]
-[![Docker Pulls](https://img.shields.io/docker/pulls/tootsuite/mastodon.svg)][docker]
 
 [releases]: https://github.com/mastodon/mastodon/releases
 [circleci]: https://circleci.com/gh/mastodon/mastodon
 [code_climate]: https://codeclimate.com/github/mastodon/mastodon
 [crowdin]: https://crowdin.com/project/mastodon
-[docker]: https://hub.docker.com/r/tootsuite/mastodon/
 
 Mastodon is a **free, open-source social network server** based on ActivityPub where users can follow friends and discover new ones. On Mastodon, users can publish anything they want: links, pictures, text, video. All Mastodon servers are interoperable as a federated network (users on one server can seamlessly communicate with users from another one, including non-Mastodon software that implements ActivityPub)!
 
@@ -28,6 +26,7 @@ Click below to **learn more** in a video:
 - [View sponsors](https://joinmastodon.org/sponsors)
 - [Blog](https://blog.joinmastodon.org)
 - [Documentation](https://docs.joinmastodon.org)
+- [Official Docker image](https://github.com/mastodon/mastodon/pkgs/container/mastodon)
 - [Browse Mastodon servers](https://joinmastodon.org/communities)
 - [Browse Mastodon apps](https://joinmastodon.org/apps)
 

app/controllers/admin/accounts_controller.rb

@@ -49,12 +49,14 @@ module Admin
     def approve
       authorize @account.user, :approve?
       @account.user.approve!
+      log_action :approve, @account.user
       redirect_to admin_accounts_path(status: 'pending'), notice: I18n.t('admin.accounts.approved_msg', username: @account.acct)
     end
 
     def reject
       authorize @account.user, :reject?
       DeleteAccountService.new.call(@account, reserve_email: false, reserve_username: false)
+      log_action :reject, @account.user
       redirect_to admin_accounts_path(status: 'pending'), notice: I18n.t('admin.accounts.rejected_msg', username: @account.acct)
     end
 

app/controllers/admin/domain_blocks_controller.rb

@@ -43,12 +43,8 @@ module Admin
     def update
       authorize :domain_block, :update?
 
-      @domain_block.update(update_params)
-
-      severity_changed = @domain_block.severity_changed?
-
-      if @domain_block.save
-        DomainBlockWorker.perform_async(@domain_block.id, severity_changed)
+      if @domain_block.update(update_params)
+        DomainBlockWorker.perform_async(@domain_block.id, @domain_block.severity_previously_changed?)
         log_action :update, @domain_block
         redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg')
       else

app/controllers/admin/instances_controller.rb

@@ -57,7 +57,7 @@ module Admin
     end
 
     def preload_delivery_failures!
-      warning_domains_map = DeliveryFailureTracker.warning_domains_map
+      warning_domains_map = DeliveryFailureTracker.warning_domains_map(@instances.map(&:domain))
 
       @instances.each do |instance|
         instance.failure_days = warning_domains_map[instance.domain]

app/controllers/api/v1/admin/accounts_controller.rb

@@ -54,12 +54,14 @@ class Api::V1::Admin::AccountsController < Api::BaseController
   def approve
     authorize @account.user, :approve?
     @account.user.approve!
+    log_action :approve, @account.user
     render json: @account, serializer: REST::Admin::AccountSerializer
   end
 
   def reject
     authorize @account.user, :reject?
     DeleteAccountService.new.call(@account, reserve_email: false, reserve_username: false)
+    log_action :reject, @account.user
     render json: @account, serializer: REST::Admin::AccountSerializer
   end
 

app/controllers/auth/sessions_controller.rb

@@ -12,6 +12,10 @@ class Auth::SessionsController < Devise::SessionsController
   before_action :set_instance_presenter, only: [:new]
   before_action :set_body_classes
 
+  content_security_policy only: :new do |p|
+    p.form_action(false)
+  end
+
   def create
     super do |resource|
       # We only need to call this if this hasn't already been

app/controllers/backups_controller.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class BackupsController < ApplicationController
+  include RoutingHelper
+
+  skip_before_action :require_functional!
+
+  before_action :authenticate_user!
+  before_action :set_backup
+
+  def download
+    case Paperclip::Attachment.default_options[:storage]
+    when :s3
+      redirect_to @backup.dump.expiring_url(10)
+    when :fog
+      redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
+    when :filesystem
+      redirect_to full_asset_url(@backup.dump.url)
+    end
+  end
+
+  private
+
+  def set_backup
+    @backup = current_user.backups.find(params[:id])
+  end
+end

app/controllers/oauth/authorizations_controller.rb

@@ -7,6 +7,10 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
   before_action :authenticate_resource_owner!
   before_action :set_cache_headers
 
+  content_security_policy do |p|
+    p.form_action(false)
+  end
+
   include Localized
 
   private

app/controllers/relationships_controller.rb

@@ -19,6 +19,8 @@ class RelationshipsController < ApplicationController
     @form.save
   rescue ActionController::ParameterMissing
     # Do nothing
+  rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound
+    flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow'
   ensure
     redirect_to relationships_path(filter_params)
   end
@@ -60,8 +62,8 @@ class RelationshipsController < ApplicationController
       'unfollow'
     elsif params[:remove_from_followers]
       'remove_from_followers'
-    elsif params[:block_domains]
-      'block_domains'
+    elsif params[:block_domains] || params[:remove_domains_from_followers]
+      'remove_domains_from_followers'
     end
   end
 

app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb

@@ -52,7 +52,7 @@ module Settings
             end
           else
             flash[:error] = I18n.t('webauthn_credentials.create.error')
-            status = :internal_server_error
+            status = :unprocessable_entity
           end
         else
           flash[:error] = t('webauthn_credentials.create.error')

app/javascript/mastodon/features/compose/components/language_dropdown.js

@@ -222,7 +222,7 @@ class LanguageDropdownMenu extends React.PureComponent {
 
     return (
       <div key={lang[0]} role='option' tabIndex='0' data-index={lang[0]} className={classNames('language-dropdown__dropdown__results__item', { active: lang[0] === value })} aria-selected={lang[0] === value} onClick={this.handleClick} onKeyDown={this.handleKeyDown}>
-        <span className='language-dropdown__dropdown__results__item__native-name'>{lang[2]}</span> <span className='language-dropdown__dropdown__results__item__common-name'>({lang[1]})</span>
+        <span className='language-dropdown__dropdown__results__item__native-name' lang={lang[0]}>{lang[2]}</span> <span className='language-dropdown__dropdown__results__item__common-name'>({lang[1]})</span>
       </div>
     );
   }

app/javascript/mastodon/features/emoji/__tests__/emoji-test.js

@@ -11,8 +11,8 @@ describe('emoji', () => {
     });
 
     it('works with unclosed tags', () => {
-      expect(emojify('hello>')).toEqual('hello>');
-      expect(emojify('<hello')).toEqual('<hello');
+      expect(emojify('hello>')).toEqual('hello&gt;');
+      expect(emojify('<hello')).toEqual('');
     });
 
     it('works with unclosed shortcodes', () => {
@@ -22,23 +22,23 @@ describe('emoji', () => {
 
     it('does unicode', () => {
       expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).toEqual(
-        '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg" />');
+        '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg">');
       expect(emojify('👨‍👩‍👧‍👧')).toEqual(
-        '<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg" />');
-      expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg" />');
+        '<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg">');
+      expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg">');
       expect(emojify('\u2757')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
     });
 
     it('does multiple unicode', () => {
       expect(emojify('\u2757 #\uFE0F\u20E3')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757#\uFE0F\u20E3')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757 #\uFE0F\u20E3 \u2757')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" /> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
       expect(emojify('foo \u2757 #\uFE0F\u20E3 bar')).toEqual(
-        'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" /> bar');
+        'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> bar');
     });
 
     it('ignores unicode inside of tags', () => {
@@ -46,16 +46,16 @@ describe('emoji', () => {
     });
 
     it('does multiple emoji properly (issue 5188)', () => {
-      expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />');
-      expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />');
+      expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
+      expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
     });
 
     it('does an emoji that has no shortcode', () => {
-      expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg" />');
+      expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg">');
     });
 
     it('does an emoji whose filename is irregular', () => {
-      expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg" />');
+      expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg">');
     });
 
     it('avoid emojifying on invisible text', () => {
@@ -67,26 +67,26 @@ describe('emoji', () => {
 
     it('avoid emojifying on invisible text with nested tags', () => {
       expect(emojify('<span class="invisible">😄<span class="foo">bar</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
+        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
       expect(emojify('<span class="invisible">😄<span class="invisible">😕</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
-      expect(emojify('<span class="invisible">😄<br/>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<br/>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
+        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
+      expect(emojify('<span class="invisible">😄<br>😴</span>😇'))
+        .toEqual('<span class="invisible">😄<br>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
     });
 
     it('skips the textual presentation VS15 character', () => {
       expect(emojify('✴︎')) // This is U+2734 EIGHT POINTED BLACK STAR then U+FE0E VARIATION SELECTOR-15
-        .toEqual('<img draggable="false" class="emojione" alt="✴" title=":eight_pointed_black_star:" src="/emoji/2734_border.svg" />');
+        .toEqual('<img draggable="false" class="emojione" alt="✴" title=":eight_pointed_black_star:" src="/emoji/2734_border.svg">');
     });
 
     it('does an simple emoji properly', () => {
       expect(emojify('♀♂'))
-        .toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg" /><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg" />');
+        .toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg">');
     });
 
     it('does an emoji containing ZWJ properly', () => {
       expect(emojify('💂‍♀️💂‍♂️'))
-        .toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg" /><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg" />');
+        .toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg">');
     });
   });
 });

app/javascript/mastodon/features/emoji/emoji.js

@@ -19,15 +19,26 @@ const emojiFilename = (filename) => {
   return borderedEmoji.includes(filename) ? (filename + '_border') : filename;
 };
 
-const emojify = (str, customEmojis = {}) => {
-  const tagCharsWithoutEmojis = '<&';
-  const tagCharsWithEmojis = Object.keys(customEmojis).length ? '<&:' : '<&';
-  let rtn = '', tagChars = tagCharsWithEmojis, invisible = 0;
+const domParser = new DOMParser();
+
+const emojifyTextNode = (node, customEmojis) => {
+  let str = node.textContent;
+
+  const fragment = new DocumentFragment();
+
   for (;;) {
-    let match, i = 0, tag;
-    while (i < str.length && (tag = tagChars.indexOf(str[i])) === -1 && (invisible || !(match = trie.search(str.slice(i))))) {
-      i += str.codePointAt(i) < 65536 ? 1 : 2;
+    let match, i = 0;
+
+    if (customEmojis === null) {
+      while (i < str.length && !(match = trie.search(str.slice(i)))) {
+        i += str.codePointAt(i) < 65536 ? 1 : 2;
+      }
+    } else {
+      while (i < str.length && str[i] !== ':' && !(match = trie.search(str.slice(i)))) {
+        i += str.codePointAt(i) < 65536 ? 1 : 2;
+      }
     }
+
     let rend, replacement = '';
     if (i === str.length) {
       break;
@@ -35,8 +46,6 @@ const emojify = (str, customEmojis = {}) => {
       if (!(() => {
         rend = str.indexOf(':', i + 1) + 1;
         if (!rend) return false; // no pair of ':'
-        const lt = str.indexOf('<', i + 1);
-        if (!(lt === -1 || lt >= rend)) return false; // tag appeared before closing ':'
         const shortname = str.slice(i, rend);
         // now got a replacee as ':shortname:'
         // if you want additional emoji handler, add statements below which set replacement and return true.
@@ -47,29 +56,6 @@ const emojify = (str, customEmojis = {}) => {
         }
         return false;
       })()) rend = ++i;
-    } else if (tag >= 0) { // <, &
-      rend = str.indexOf('>;'[tag], i + 1) + 1;
-      if (!rend) {
-        break;
-      }
-      if (tag === 0) {
-        if (invisible) {
-          if (str[i + 1] === '/') { // closing tag
-            if (!--invisible) {
-              tagChars = tagCharsWithEmojis;
-            }
-          } else if (str[rend - 2] !== '/') { // opening tag
-            invisible++;
-          }
-        } else {
-          if (str.startsWith('<span class="invisible">', i)) {
-            // avoid emojifying on invisible text
-            invisible = 1;
-            tagChars = tagCharsWithoutEmojis;
-          }
-        }
-      }
-      i = rend;
     } else { // matched to unicode emoji
       const { filename, shortCode } = unicodeMapping[match];
       const title = shortCode ? `:${shortCode}:` : '';
@@ -80,10 +66,43 @@ const emojify = (str, customEmojis = {}) => {
         rend += 1;
       }
     }
-    rtn += str.slice(0, i) + replacement;
+
+    fragment.append(document.createTextNode(str.slice(0, i)));
+    if (replacement) {
+      fragment.append(domParser.parseFromString(replacement, 'text/html').documentElement.getElementsByTagName('img')[0]);
+    }
+    node.textContent = str.slice(0, i);
     str = str.slice(rend);
   }
-  return rtn + str;
+
+  fragment.append(document.createTextNode(str));
+  node.parentElement.replaceChild(fragment, node);
+};
+
+const emojifyNode = (node, customEmojis) => {
+  for (const child of node.childNodes) {
+    switch(child.nodeType) {
+    case Node.TEXT_NODE:
+      emojifyTextNode(child, customEmojis);
+      break;
+    case Node.ELEMENT_NODE:
+      if (!child.classList.contains('invisible'))
+        emojifyNode(child, customEmojis);
+      break;
+    }
+  }
+};
+
+const emojify = (str, customEmojis = {}) => {
+  const wrapper = document.createElement('div');
+  wrapper.innerHTML = str;
+
+  if (!Object.keys(customEmojis).length)
+    customEmojis = null;
+
+  emojifyNode(wrapper, customEmojis);
+
+  return wrapper.innerHTML;
 };
 
 export default emojify;

app/javascript/mastodon/reducers/compose.js

@@ -184,11 +184,12 @@ const ignoreSuggestion = (state, position, token, completion, path) => {
 };
 
 const sortHashtagsByUse = (state, tags) => {
-  const personalHistory = state.get('tagHistory');
+  const personalHistory = state.get('tagHistory').map(tag => tag.toLowerCase());
 
-  return tags.sort((a, b) => {
-    const usedA = personalHistory.includes(a.name);
-    const usedB = personalHistory.includes(b.name);
+  const tagsWithLowercase = tags.map(t => ({ ...t, lowerName: t.name.toLowerCase() }));
+  const sorted = tagsWithLowercase.sort((a, b) => {
+    const usedA = personalHistory.includes(a.lowerName);
+    const usedB = personalHistory.includes(b.lowerName);
 
     if (usedA === usedB) {
       return 0;
@@ -198,6 +199,8 @@ const sortHashtagsByUse = (state, tags) => {
       return 1;
     }
   });
+  sorted.forEach(tag => delete tag.lowerName);
+  return sorted;
 };
 
 const insertEmoji = (state, position, emojiData, needsSpace) => {

app/javascript/styles/mastodon/components.scss

@@ -4261,6 +4261,7 @@ a.status-card.compact:hover {
   display: flex;
   align-items: center;
   justify-content: center;
+  text-align: center;
   color: $secondary-text-color;
   font-size: 18px;
   font-weight: 500;

app/lib/activitypub/activity.rb

@@ -106,7 +106,8 @@ class ActivityPub::Activity
       actor_id = value_or_id(first_of_value(@object['attributedTo']))
 
       if actor_id == @account.uri
-        return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform
+        virtual_object = { 'type' => 'Create', 'actor' => actor_id, 'object' => @object }
+        return ActivityPub::Activity.factory(virtual_object, @account, request_id: @options[:request_id]).perform
       end
     end
 
@@ -152,9 +153,9 @@ class ActivityPub::Activity
   def fetch_remote_original_status
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first)
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
     elsif @object['url'].present?
-      ::FetchRemoteStatusService.new.call(@object['url'])
+      ::FetchRemoteStatusService.new.call(@object['url'], request_id: @options[:request_id])
     end
   end
 

app/lib/activitypub/activity/create.rb

@@ -222,7 +222,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if tag['href'].blank?
 
     account = account_from_uri(tag['href'])
-    account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
+    account = ActivityPub::FetchRemoteAccountService.new.call(tag['href'], request_id: @options[:request_id]) if account.nil?
 
     return if account.nil?
 
@@ -327,18 +327,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def resolve_thread(status)
     return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
 
-    ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
+    ThreadResolveWorker.perform_async(status.id, in_reply_to_uri, { 'request_id' => @options[:request_id]})
   end
 
   def fetch_replies(status)
     collection = @object['replies']
     return if collection.nil?
 
-    replies = ActivityPub::FetchRepliesService.new.call(status, collection, false)
+    replies = ActivityPub::FetchRepliesService.new.call(status, collection, allow_synchronous_requests: false, request_id: @options[:request_id])
     return unless replies.nil?
 
     uri = value_or_id(collection)
-    ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
+    ActivityPub::FetchRepliesWorker.perform_async(status.id, uri, { 'request_id' => @options[:request_id]}) unless uri.nil?
   end
 
   def conversation_from_uri(uri)

app/lib/activitypub/activity/update.rb

@@ -18,7 +18,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
   def update_account
     return reject_payload! if @account.uri != object_uri
 
-    ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true)
+    ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true, request_id: @options[:request_id])
   end
 
   def update_status
@@ -28,6 +28,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
 
     return if @status.nil?
 
-    ActivityPub::ProcessStatusUpdateService.new.call(@status, @object)
+    ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id])
   end
 end

app/lib/admin/system_check.rb

@@ -2,6 +2,7 @@
 
 class Admin::SystemCheck
   ACTIVE_CHECKS = [
+    Admin::SystemCheck::MediaPrivacyCheck,
     Admin::SystemCheck::DatabaseSchemaCheck,
     Admin::SystemCheck::SidekiqProcessCheck,
     Admin::SystemCheck::RulesCheck,

app/lib/admin/system_check/elasticsearch_check.rb

@@ -20,7 +20,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
   def running_version
     @running_version ||= begin
       Chewy.client.info['version']['number']
-    rescue Faraday::ConnectionFailed
+    rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
       nil
     end
   end

app/lib/admin/system_check/media_privacy_check.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck
+  include RoutingHelper
+
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
+  def pass?
+    check_media_uploads!
+    @failure_message.nil?
+  end
+
+  def message
+    Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true)
+  end
+
+  private
+
+  def check_media_uploads!
+    if Rails.configuration.x.use_s3
+      check_media_listing_inaccessible_s3!
+    else
+      check_media_listing_inaccessible!
+    end
+  end
+
+  def check_media_listing_inaccessible!
+    full_url = full_asset_url(media_attachment.file.url(:original, false))
+
+    # Check if we can list the uploaded file. If true, that's an error
+    directory_url = Addressable::URI.parse(full_url)
+    directory_url.query = nil
+    filename = directory_url.path.gsub(%r{.*/}, '')
+    directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/')
+    Request.new(:get, directory_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?(filename)
+        @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error
+        @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS'
+      end
+    end
+  rescue
+    nil
+  end
+
+  def check_media_listing_inaccessible_s3!
+    urls_to_check = []
+    paperclip_options = Paperclip::Attachment.default_options
+    s3_protocol = paperclip_options[:s3_protocol]
+    s3_host_alias = paperclip_options[:s3_host_alias]
+    s3_host_name  = paperclip_options[:s3_host_name]
+    bucket_name = paperclip_options.dig(:s3_credentials, :bucket)
+
+    urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present?
+    urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/"
+    urls_to_check.uniq.each do |full_url|
+      check_s3_listing!(full_url)
+      break if @failure_message.present?
+    end
+  rescue
+    nil
+  end
+
+  def check_s3_listing!(full_url)
+    bucket_url = Addressable::URI.parse(full_url)
+    bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original))
+    bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}"
+    Request.new(:get, bucket_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?('ListBucketResult')
+        @failure_message = :upload_check_privacy_error_object_storage
+        @failure_action  = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3'
+      end
+    end
+  end
+
+  def media_attachment
+    @media_attachment ||= begin
+      attachment = Account.representative.media_attachments.first
+      if attachment.present?
+        attachment.touch # rubocop:disable Rails/SkipsModelValidations
+        attachment
+      else
+        create_test_attachment!
+      end
+    end
+  end
+
+  def create_test_attachment!
+    Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file|
+      tmp_file.write(
+        Base64.decode64(
+          '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \
+          'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \
+          'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \
+          'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \
+          'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \
+          'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q=='
+        )
+      )
+      tmp_file.flush
+      Account.representative.media_attachments.create!(file: tmp_file)
+    end
+  end
+end

app/lib/admin/system_check/message.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::Message
-  attr_reader :key, :value, :action
+  attr_reader :key, :value, :action, :critical
 
-  def initialize(key, value = nil, action = nil)
-    @key    = key
-    @value  = value
-    @action = action
+  def initialize(key, value = nil, action = nil, critical = false)
+    @key      = key
+    @value    = value
+    @action   = action
+    @critical = critical
   end
 end

app/lib/delivery_failure_tracker.rb

@@ -65,8 +65,13 @@ class DeliveryFailureTracker
       domains - UnavailableDomain.all.pluck(:domain)
     end
 
-    def warning_domains_map
-      warning_domains.index_with { |domain| redis.scard(exhausted_deliveries_key_by(domain)) }
+    def warning_domains_map(domains = nil)
+      if domains.nil?
+        warning_domains.index_with { |domain| redis.scard(exhausted_deliveries_key_by(domain)) }
+      else
+        domains -= UnavailableDomain.where(domain: domains).pluck(:domain)
+        domains.index_with { |domain| redis.scard(exhausted_deliveries_key_by(domain)) }.filter { |_, days| days.positive? }
+      end
     end
 
     private

app/lib/emoji_formatter.rb

@@ -23,48 +23,40 @@ class EmojiFormatter
   def to_s
     return html if custom_emojis.empty? || html.blank?
 
-    i                     = -1
-    tag_open_index        = nil
-    inside_shortname      = false
-    shortname_start_index = -1
-    invisible_depth       = 0
-    last_index            = 0
-    result                = ''.dup
+    tree = Nokogiri::HTML.fragment(html)
+    tree.xpath('./text()|.//text()[not(ancestor[@class="invisible"])]').to_a.each do |node|
+      i                     = -1
+      inside_shortname      = false
+      shortname_start_index = -1
+      last_index            = 0
+      text                  = node.content
+      result                = Nokogiri::XML::NodeSet.new(tree.document)
 
-    while i + 1 < html.size
-      i += 1
+      while i + 1 < text.size
+        i += 1
 
-      if invisible_depth.zero? && inside_shortname && html[i] == ':'
-        inside_shortname = false
-        shortcode = html[shortname_start_index + 1..i - 1]
-        char_after = html[i + 1]
+        if inside_shortname && text[i] == ':'
+          inside_shortname = false
+          shortcode = text[shortname_start_index + 1..i - 1]
+          char_after = text[i + 1]
 
-        next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode])
+          next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode])
 
-        result << html[last_index..shortname_start_index - 1] if shortname_start_index.positive?
-        result << image_for_emoji(shortcode, emoji)
-        last_index = i + 1
-      elsif tag_open_index && html[i] == '>'
-        tag = html[tag_open_index..i]
-        tag_open_index = nil
+          result << Nokogiri::XML::Text.new(text[last_index..shortname_start_index - 1], tree.document) if shortname_start_index.positive?
+          result << Nokogiri::HTML.fragment(image_for_emoji(shortcode, emoji))
 
-        if invisible_depth.positive?
-          invisible_depth += count_tag_nesting(tag)
-        elsif tag == '<span class="invisible">'
-          invisible_depth = 1
+          last_index = i + 1
+        elsif text[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(text[i - 1]))
+          inside_shortname = true
+          shortname_start_index = i
         end
-      elsif html[i] == '<'
-        tag_open_index = i
-        inside_shortname = false
-      elsif !tag_open_index && html[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(html[i - 1]))
-        inside_shortname = true
-        shortname_start_index = i
       end
+
+      result << Nokogiri::XML::Text.new(text[last_index..-1], tree.document)
+      node.replace(result)
     end
 
-    result << html[last_index..-1]
-
-    result.html_safe # rubocop:disable Rails/OutputSafety
+    tree.to_html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
   private

app/lib/status_reach_finder.rb

@@ -70,7 +70,7 @@ class StatusReachFinder
 
   def followers_inboxes
     if @status.in_reply_to_local_account? && distributable?
-      @status.account.followers.or(@status.thread.account.followers).inboxes
+      @status.account.followers.or(@status.thread.account.followers.not_domain_blocked_by_account(@status.account)).inboxes
     elsif @status.direct_visibility? || @status.limited_visibility?
       []
     else

app/mailers/application_mailer.rb

@@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base
   helper :instance
   helper :formatting
 
+  after_action :set_autoreply_headers!
+
   protected
 
   def locale_for_account(account)
@@ -14,4 +16,10 @@ class ApplicationMailer < ActionMailer::Base
       yield
     end
   end
+
+  def set_autoreply_headers!
+    headers['Precedence'] = 'list'
+    headers['X-Auto-Response-Suppress'] = 'All'
+    headers['Auto-Submitted'] = 'auto-generated'
+  end
 end

app/models/account.rb

@@ -106,7 +106,7 @@ class Account < ApplicationRecord
   scope :bots, -> { where(actor_type: %w(Application Service)) }
   scope :groups, -> { where(actor_type: 'Group') }
   scope :alphabetic, -> { order(domain: :asc, username: :asc) }
-  scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) }
+  scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") }
   scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) }
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) }

app/models/admin/status_batch_action.rb

@@ -73,7 +73,7 @@ class Admin::StatusBatchAction
     # Can't use a transaction here because UpdateStatusService queues
     # Sidekiq jobs
     statuses.includes(:media_attachments, :preview_cards).find_each do |status|
-      next unless status.with_media? || status.with_preview_card?
+      next if status.discarded? || !(status.with_media? || status.with_preview_card?)
 
       authorize(status, :update?)
 
@@ -89,15 +89,15 @@ class Admin::StatusBatchAction
         report.resolve!(current_account)
         log_action(:resolve, report)
       end
-
-      @warning = target_account.strikes.create!(
-        action: :mark_statuses_as_sensitive,
-        account: current_account,
-        report: report,
-        status_ids: status_ids
-      )
     end
 
+    @warning = target_account.strikes.create!(
+      action: :mark_statuses_as_sensitive,
+      account: current_account,
+      report: report,
+      status_ids: status_ids
+    )
+
     UserMailer.warning(target_account.user, @warning).deliver_later! if warnable?
   end
 

app/models/backup.rb

@@ -17,6 +17,6 @@
 class Backup < ApplicationRecord
   belongs_to :user, inverse_of: :backups
 
-  has_attached_file :dump
+  has_attached_file :dump, s3_permissions: 'private'
   do_not_validate_attachment_file_type :dump
 end

app/models/concerns/domain_materializable.rb

@@ -3,11 +3,24 @@
 module DomainMaterializable
   extend ActiveSupport::Concern
 
+  include Redisable
+
   included do
     after_create_commit :refresh_instances_view
   end
 
   def refresh_instances_view
-    Instance.refresh unless domain.nil? || Instance.where(domain: domain).exists?
+    return if domain.nil? || Instance.exists?(domain: domain)
+
+    Instance.refresh
+    count_unique_subdomains!
+  end
+
+  def count_unique_subdomains!
+    second_and_top_level_domain = PublicSuffix.domain(domain, ignore_private: true)
+    with_redis do |redis|
+      redis.pfadd("unique_subdomains_for:#{second_and_top_level_domain}", domain)
+      redis.expire("unique_subdomains_for:#{second_and_top_level_domain}", 1.minute.seconds)
+    end
   end
 end

app/models/form/account_batch.rb

@@ -16,8 +16,8 @@ class Form::AccountBatch
       unfollow!
     when 'remove_from_followers'
       remove_from_followers!
-    when 'block_domains'
-      block_domains!
+    when 'remove_domains_from_followers'
+      remove_domains_from_followers!
     when 'approve'
       approve!
     when 'reject'
@@ -34,9 +34,15 @@ class Form::AccountBatch
   private
 
   def follow!
+    error = nil
+
     accounts.each do |target_account|
       FollowService.new.call(current_account, target_account)
+    rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e
+      error ||= e
     end
+
+    raise error if error.present?
   end
 
   def unfollow!
@@ -49,10 +55,8 @@ class Form::AccountBatch
     RemoveFromFollowersService.new.call(current_account, account_ids)
   end
 
-  def block_domains!
-    AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain|
-      [current_account.id, domain]
-    end
+  def remove_domains_from_followers!
+    RemoveDomainsFromFollowersService.new.call(current_account, account_domains)
   end
 
   def account_domains

app/models/poll.rb

@@ -85,6 +85,7 @@ class Poll < ApplicationRecord
   def reset_votes!
     self.cached_tallies = options.map { 0 }
     self.votes_count = 0
+    self.voters_count = 0
     votes.delete_all unless new_record?
   end
 

app/models/user.rb

@@ -442,10 +442,13 @@ class User < ApplicationRecord
   def prepare_new_user!
     BootstrapTimelineWorker.perform_async(account_id)
     ActivityTracker.increment('activity:accounts:local')
+    ActivityTracker.record('activity:logins', id)
     UserMailer.welcome(self).deliver_later
   end
 
   def prepare_returning_user!
+    return unless confirmed?
+
     ActivityTracker.record('activity:logins', id)
     regenerate_feed! if needs_feed_update?
   end

app/serializers/rest/account_serializer.rb

@@ -15,6 +15,16 @@ class REST::AccountSerializer < ActiveModel::Serializer
   attribute :suspended, if: :suspended?
   attribute :silenced, key: :limited, if: :silenced?
 
+  class AccountDecorator < SimpleDelegator
+    def self.model_name
+      Account.model_name
+    end
+
+    def moved?
+      false
+    end
+  end
+
   class FieldSerializer < ActiveModel::Serializer
     include FormattingHelper
 
@@ -84,7 +94,7 @@ class REST::AccountSerializer < ActiveModel::Serializer
   end
 
   def moved_to_account
-    object.suspended? ? nil : object.moved_to_account
+    object.suspended? ? nil : AccountDecorator.new(object.moved_to_account)
   end
 
   def emojis
@@ -106,6 +116,6 @@ class REST::AccountSerializer < ActiveModel::Serializer
   delegate :suspended?, :silenced?, to: :object
 
   def moved_and_not_nested?
-    object.moved? && object.moved_to_account.moved_to_account_id.nil?
+    object.moved?
   end
 end

app/services/activitypub/fetch_featured_collection_service.rb

@@ -3,10 +3,11 @@
 class ActivityPub::FetchFeaturedCollectionService < BaseService
   include JsonLdHelper
 
-  def call(account)
+  def call(account, **options)
     return if account.featured_collection_url.blank? || account.suspended? || account.local?
 
     @account = account
+    @options = options
     @json    = fetch_resource(@account.featured_collection_url, true, local_follower)
 
     return unless supported_context?(@json)
@@ -38,9 +39,9 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
   def process_items(items)
     status_ids = items.filter_map do |item|
       uri = value_or_id(item)
-      next if ActivityPub::TagManager.instance.local_uri?(uri)
+      next if ActivityPub::TagManager.instance.local_uri?(uri) || invalid_origin?(uri)
 
-      status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower)
+      status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower, expected_actor_uri: @account.uri, request_id: @options[:request_id])
       next unless status&.account_id == @account.id
 
       status.id

app/services/activitypub/fetch_remote_account_service.rb

@@ -8,7 +8,7 @@ class ActivityPub::FetchRemoteAccountService < BaseService
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
 
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false)
+  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, request_id: nil)
     return if domain_not_allowed?(uri)
     return ActivityPub::TagManager.instance.uri_to_resource(uri, Account) if ActivityPub::TagManager.instance.local_uri?(uri)
 
@@ -28,7 +28,7 @@ class ActivityPub::FetchRemoteAccountService < BaseService
 
     return unless only_key || verified_webfinger?
 
-    ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key)
+    ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key, request_id: request_id)
   rescue Oj::ParseError
     nil
   end

app/services/activitypub/fetch_remote_status_service.rb

@@ -2,9 +2,13 @@
 
 class ActivityPub::FetchRemoteStatusService < BaseService
   include JsonLdHelper
+  include Redisable
+
+  DISCOVERIES_PER_REQUEST = 1000
 
   # Should be called when uri has already been checked for locality
-  def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil)
+  def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
+    @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}"
     @json = begin
       if prefetched_body.nil?
         fetch_resource(uri, id, on_behalf_of)
@@ -30,6 +34,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     end
 
     return if activity_json.nil? || object_uri.nil? || !trustworthy_attribution?(@json['id'], actor_uri)
+    return if expected_actor_uri.present? && actor_uri != expected_actor_uri
     return ActivityPub::TagManager.instance.uri_to_resource(object_uri, Status) if ActivityPub::TagManager.instance.local_uri?(object_uri)
 
     actor = account_from_uri(actor_uri)
@@ -40,7 +45,13 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     # activity as an update rather than create
     activity_json['type'] = 'Update' if equals_or_includes_any?(activity_json['type'], %w(Create)) && Status.where(uri: object_uri, account_id: actor.id).exists?
 
-    ActivityPub::Activity.factory(activity_json, actor).perform
+    with_redis do |redis|
+      discoveries = redis.incr("status_discovery_per_request:#{@request_id}")
+      redis.expire("status_discovery_per_request:#{@request_id}", 5.minutes.seconds)
+      return nil if discoveries > DISCOVERIES_PER_REQUEST
+    end
+
+    ActivityPub::Activity.factory(activity_json, actor, request_id: @request_id).perform
   end
 
   private
@@ -52,7 +63,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
   def account_from_uri(uri)
     actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
-    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true) if actor.nil? || actor.possibly_stale?
+    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true, request_id: @request_id) if actor.nil? || actor.possibly_stale?
     actor
   end
 

app/services/activitypub/fetch_replies_service.rb

@@ -3,14 +3,14 @@
 class ActivityPub::FetchRepliesService < BaseService
   include JsonLdHelper
 
-  def call(parent_status, collection_or_uri, allow_synchronous_requests = true)
+  def call(parent_status, collection_or_uri, allow_synchronous_requests: true, request_id: nil)
     @account = parent_status.account
     @allow_synchronous_requests = allow_synchronous_requests
 
     @items = collection_items(collection_or_uri)
     return if @items.nil?
 
-    FetchReplyWorker.push_bulk(filtered_replies)
+    FetchReplyWorker.push_bulk(filtered_replies) { |reply_uri| [reply_uri, { 'request_id' => request_id}] }
 
     @items
   end

app/services/activitypub/process_account_service.rb

@@ -6,6 +6,9 @@ class ActivityPub::ProcessAccountService < BaseService
   include Redisable
   include Lockable
 
+  SUBDOMAINS_RATELIMIT = 10
+  DISCOVERIES_PER_REQUEST = 400
+
   # Should be called with confirmed valid JSON
   # and WebFinger-resolved username and domain
   def call(username, domain, json, options = {})
@@ -15,9 +18,12 @@ class ActivityPub::ProcessAccountService < BaseService
     @json        = json
     @uri         = @json['id']
     @username    = username
-    @domain      = domain
+    @domain      = TagManager.instance.normalize_domain(domain)
     @collections = {}
 
+    # The key does not need to be unguessable, it just needs to be somewhat unique
+    @options[:request_id] ||= "#{Time.now.utc.to_i}-#{username}@#{domain}"
+
     with_lock("process_account:#{@uri}") do
       @account            = Account.remote.find_by(uri: @uri) if @options[:only_key]
       @account          ||= Account.find_remote(@username, @domain)
@@ -25,7 +31,18 @@ class ActivityPub::ProcessAccountService < BaseService
       @old_protocol       = @account&.protocol
       @suspension_changed = false
 
-      create_account if @account.nil?
+      if @account.nil?
+        with_redis do |redis|
+          return nil if redis.pfcount("unique_subdomains_for:#{PublicSuffix.domain(@domain, ignore_private: true)}") >= SUBDOMAINS_RATELIMIT
+
+          discoveries = redis.incr("discovery_per_request:#{@options[:request_id]}")
+          redis.expire("discovery_per_request:#{@options[:request_id]}", 5.minutes.seconds)
+          return nil if discoveries > DISCOVERIES_PER_REQUEST
+        end
+
+        create_account
+      end
+
       update_account
       process_tags
 
@@ -149,7 +166,7 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def check_featured_collection!
-    ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id)
+    ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'request_id' => @options[:request_id] })
   end
 
   def check_links!
@@ -249,7 +266,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def moved_account
     account   = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account)
-    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true)
+    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true, request_id: @options[:request_id])
     account
   end
 

app/services/activitypub/process_status_update_service.rb

@@ -5,7 +5,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   include Redisable
   include Lockable
 
-  def call(status, json)
+  def call(status, json, request_id: nil)
     raise ArgumentError, 'Status has unsaved changes' if status.changed?
 
     @json                      = json
@@ -15,6 +15,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     @account                   = status.account
     @media_attachments_changed = false
     @poll_changed              = false
+    @request_id                = request_id
 
     # Only native types can be updated at the moment
     return @status if !expected_type? || already_updated_more_recently?
@@ -92,7 +93,13 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
 
         next if unsupported_media_type?(media_attachment_parser.file_content_type) || skip_download?
 
-        RedownloadMediaWorker.perform_async(media_attachment.id) if media_attachment.remote_url_previously_changed? || media_attachment.thumbnail_remote_url_previously_changed?
+        begin
+          media_attachment.download_file! if media_attachment.remote_url_previously_changed?
+          media_attachment.download_thumbnail! if media_attachment.thumbnail_remote_url_previously_changed?
+          media_attachment.save
+        rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
+          RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
+        end
       rescue Addressable::URI::InvalidURIError => e
         Rails.logger.debug "Invalid URL in attachment: #{e}"
       end
@@ -185,7 +192,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
       next if href.blank?
 
       account   = ActivityPub::TagManager.instance.uri_to_resource(href, Account)
-      account ||= ActivityPub::FetchRemoteAccountService.new.call(href)
+      account ||= ActivityPub::FetchRemoteAccountService.new.call(href, request_id: @request_id)
 
       next if account.nil?
 

app/services/fetch_remote_status_service.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class FetchRemoteStatusService < BaseService
-  def call(url, prefetched_body = nil)
+  def call(url, prefetched_body: nil, request_id: nil)
     if prefetched_body.nil?
       resource_url, resource_options = FetchResourceService.new.call(url)
     else
@@ -9,6 +9,6 @@ class FetchRemoteStatusService < BaseService
       resource_options = { prefetched_body: prefetched_body }
     end
 
-    ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options) unless resource_url.nil?
+    ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options.merge(request_id: request_id)) unless resource_url.nil?
   end
 end

app/services/follow_migration_service.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class FollowMigrationService < FollowService
+  # Follow an account with the same settings as another account, and unfollow the old account once the request is sent
+  # @param [Account] source_account From which to follow
+  # @param [Account] target_account Account to follow
+  # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one
+  # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked
+  def call(source_account, target_account, old_target_account, bypass_locked: false)
+    @old_target_account = old_target_account
+
+    follow    = source_account.active_relationships.find_by(target_account: old_target_account)
+    reblogs   = follow&.show_reblogs?
+    notify    = follow&.notify?
+
+    super(source_account, target_account, reblogs: reblogs, notify: notify, bypass_locked: bypass_locked, bypass_limit: true)
+  end
+
+  private
+
+  def request_follow!
+    follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit]))
+
+    if @target_account.local?
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
+      UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    elsif @target_account.activitypub?
+      ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id)
+    end
+
+    follow_request
+  end
+
+  def direct_follow!
+    follow = super
+    UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    follow
+  end
+
+  def follow_options
+    @options.slice(:reblogs, :notify)
+  end
+end

app/services/remove_domains_from_followers_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class RemoveDomainsFromFollowersService < BaseService
+  include Payloadable
+
+  def call(source_account, target_domains)
+    source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow|
+      follow.destroy
+
+      create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub?
+    end
+  end
+
+  private
+
+  def create_notification(follow)
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url)
+  end
+
+  def build_json(follow)
+    Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
+  end
+end

app/services/report_service.rb

@@ -57,7 +57,16 @@ class ReportService < BaseService
   end
 
   def reported_status_ids
-    AccountStatusesFilter.new(@target_account, @source_account).results.with_discarded.find(Array(@status_ids)).pluck(:id)
+    return AccountStatusesFilter.new(@target_account, @source_account).results.with_discarded.find(Array(@status_ids)).pluck(:id) if @source_account.local?
+
+    # If the account making reports is remote, it is likely anonymized so we have to relax the requirements for attaching statuses.
+    domain = @source_account.domain.to_s.downcase
+    has_followers = @target_account.followers.where(Account.arel_table[:domain].lower.eq(domain)).exists?
+    visibility = has_followers ? %i(public unlisted private) : %i(public unlisted)
+    scope = @target_account.statuses.with_discarded
+    scope.merge!(scope.where(visibility: visibility).or(scope.where('EXISTS (SELECT 1 FROM mentions m JOIN accounts a ON m.account_id = a.id WHERE lower(a.domain) = ?)', domain)))
+    # Allow missing posts to not drop reports that include e.g. a deleted post
+    scope.where(id: Array(@status_ids)).pluck(:id)
   end
 
   def payload

app/services/resolve_url_service.rb

@@ -23,7 +23,7 @@ class ResolveURLService < BaseService
     if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES)
       ActivityPub::FetchRemoteAccountService.new.call(resource_url, prefetched_body: body)
     elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
-      status = FetchRemoteStatusService.new.call(resource_url, body)
+      status = FetchRemoteStatusService.new.call(resource_url, prefetched_body: body)
       authorize_with @on_behalf_of, status, :show? unless status.nil?
       status
     end

app/services/suspend_account_service.rb

@@ -3,10 +3,13 @@
 class SuspendAccountService < BaseService
   include Payloadable
 
+  # Carry out the suspension of a recently-suspended account
+  # @param [Account] account Account to suspend
   def call(account)
+    return unless account.suspended?
+
     @account = account
 
-    suspend!
     reject_remote_follows!
     distribute_update_actor!
     unmerge_from_home_timelines!
@@ -16,10 +19,6 @@ class SuspendAccountService < BaseService
 
   private
 
-  def suspend!
-    @account.suspend! unless @account.suspended?
-  end
-
   def reject_remote_follows!
     return if @account.local? || !@account.activitypub?
 
@@ -76,10 +75,15 @@ class SuspendAccountService < BaseService
         styles.each do |style|
           case Paperclip::Attachment.default_options[:storage]
           when :s3
+            # Prevent useless S3 calls if ACLs are disabled
+            next if ENV['S3_PERMISSION'] == ''
+
             begin
               attachment.s3_object(style).acl.put(acl: 'private')
             rescue Aws::S3::Errors::NoSuchKey
               Rails.logger.warn "Tried to change acl on non-existent key #{attachment.s3_object(style).key}"
+            rescue Aws::S3::Errors::NotImplemented => e
+              Rails.logger.error "Error trying to change ACL on #{attachment.s3_object(style).key}: #{e.message}"
             end
           when :fog
             # Not supported

app/services/unsuspend_account_service.rb

@@ -2,10 +2,12 @@
 
 class UnsuspendAccountService < BaseService
   include Payloadable
+
+  # Restores a recently-unsuspended account
+  # @param [Account] account Account to restore
   def call(account)
     @account = account
 
-    unsuspend!
     refresh_remote_account!
 
     return if @account.nil? || @account.suspended?
@@ -18,10 +20,6 @@ class UnsuspendAccountService < BaseService
 
   private
 
-  def unsuspend!
-    @account.unsuspend! if @account.suspended?
-  end
-
   def refresh_remote_account!
     return if @account.local?
 
@@ -73,10 +71,15 @@ class UnsuspendAccountService < BaseService
         styles.each do |style|
           case Paperclip::Attachment.default_options[:storage]
           when :s3
+            # Prevent useless S3 calls if ACLs are disabled
+            next if ENV['S3_PERMISSION'] == ''
+
             begin
               attachment.s3_object(style).acl.put(acl: Paperclip::Attachment.default_options[:s3_permissions])
             rescue Aws::S3::Errors::NoSuchKey
               Rails.logger.warn "Tried to change acl on non-existent key #{attachment.s3_object(style).key}"
+            rescue Aws::S3::Errors::NotImplemented => e
+              Rails.logger.error "Error trying to change ACL on #{attachment.s3_object(style).key}: #{e.message}"
             end
           when :fog
             # Not supported

app/views/admin/dashboard/index.html.haml

@@ -12,7 +12,7 @@
 - unless @system_checks.empty?
   .flash-message-stack
     - @system_checks.each do |message|
-      .flash-message.warning
+      .flash-message{ class: message.critical ? 'alert' : 'warning' }
         = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil)
         - if message.action
           = link_to t("admin.system_checks.#{message.key}.action"), message.action

app/views/admin/reports/_actions.html.haml

@@ -5,7 +5,7 @@
         = link_to t('admin.reports.mark_as_resolved'), resolve_admin_report_path(@report), method: :post, class: 'button'
       .report-actions__item__description
         = t('admin.reports.actions.resolve_description_html')
-    - if @statuses.any? { |status| status.with_media? || status.with_preview_card? }
+    - if @statuses.any? { |status| (status.with_media? || status.with_preview_card?) && !status.discarded? }
       .report-actions__item
         .report-actions__item__button
           = button_tag t('admin.reports.mark_as_sensitive'), name: :mark_as_sensitive, class: 'button'

app/views/disputes/strikes/show.html.haml

@@ -50,17 +50,18 @@
             .strike-card__statuses-list__item
               - if (status = status_map[status_id.to_i])
                 .one-liner
-                  = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do
-                    = one_line_preview(status)
+                  .emojify= one_line_preview(status)
 
-                    - status.ordered_media_attachments.each do |media_attachment|
-                      %abbr{ title: media_attachment.description }
-                        = fa_icon 'link'
-                        = media_attachment.file_file_name
+                  - status.ordered_media_attachments.each do |media_attachment|
+                    %abbr{ title: media_attachment.description }
+                      = fa_icon 'link'
+                      = media_attachment.file_file_name
                 .strike-card__statuses-list__item__meta
-                  %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
-                  ·
-                  = status.application.name
+                  = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do
+                    %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+                  - unless status.application.nil?
+                    ·
+                    = status.application.name
               - else
                 .one-liner= t('disputes.strikes.status', id: status_id)
                 .strike-card__statuses-list__item__meta

app/views/relationships/show.html.haml

@@ -48,7 +48,7 @@
 
         = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } unless following_relationship?
 
-        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
+        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
     .batch-table__body
       - if @accounts.empty?
         = nothing_here 'nothing-here--under-tabs'

app/views/settings/exports/show.html.haml

@@ -64,6 +64,6 @@
             %td= l backup.created_at
             - if backup.processed?
               %td= number_to_human_size backup.dump_file_size
-              %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url
+              %td= table_link_to 'download', t('exports.archive_takeout.download'), download_backup_url(backup)
             - else
               %td{ colspan: 2 }= t('exports.archive_takeout.in_progress')

app/views/user_mailer/backup_ready.html.haml

@@ -55,5 +55,5 @@
                             %tbody
                               %tr
                                 %td.button-primary
-                                  = link_to full_asset_url(@backup.dump.url) do
+                                  = link_to download_backup_url(@backup) do
                                     %span= t 'exports.archive_takeout.download'

app/views/user_mailer/backup_ready.text.erb

@@ -4,4 +4,4 @@
 
 <%= t 'user_mailer.backup_ready.explanation' %>
 
-=> <%= full_asset_url(@backup.dump.url) %>
+=> <%= download_backup_url(@backup) %>

app/workers/activitypub/fetch_replies_worker.rb

@@ -6,8 +6,8 @@ class ActivityPub::FetchRepliesWorker
 
   sidekiq_options queue: 'pull', retry: 3
 
-  def perform(parent_status_id, replies_uri)
-    ActivityPub::FetchRepliesService.new.call(Status.find(parent_status_id), replies_uri)
+  def perform(parent_status_id, replies_uri, options = {})
+    ActivityPub::FetchRepliesService.new.call(Status.find(parent_status_id), replies_uri, **options.deep_symbolize_keys)
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/activitypub/migrated_follow_delivery_worker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker
+  def perform(json, source_account_id, inbox_url, old_target_account_id, options = {})
+    super(json, source_account_id, inbox_url, options)
+    unfollow_old_account!(old_target_account_id)
+  end
+
+  private
+
+  def unfollow_old_account!(old_target_account_id)
+    old_target_account = Account.find(old_target_account_id)
+    UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true)
+  rescue StandardError
+    true
+  end
+end

app/workers/activitypub/synchronize_featured_collection_worker.rb

@@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker
 
   sidekiq_options queue: 'pull', lock: :until_executed
 
-  def perform(account_id)
-    ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id))
+  def perform(account_id, options = {})
+    options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys)
+
+    ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options)
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/fetch_reply_worker.rb

@@ -6,7 +6,7 @@ class FetchReplyWorker
 
   sidekiq_options queue: 'pull', retry: 3
 
-  def perform(child_url)
-    FetchRemoteStatusService.new.call(child_url)
+  def perform(child_url, options = {})
+    FetchRemoteStatusService.new.call(child_url, **options.deep_symbolize_keys)
   end
 end

app/workers/scheduler/user_cleanup_scheduler.rb

@@ -15,6 +15,8 @@ class Scheduler::UserCleanupScheduler
 
   def clean_unconfirmed_accounts!
     User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).reorder(nil).find_in_batches do |batch|
+      # We have to do it separately because of missing database constraints
+      AccountModerationNote.where(target_account_id: batch.map(&:account_id)).delete_all
       Account.where(id: batch.map(&:account_id)).delete_all
       User.where(id: batch.map(&:id)).delete_all
     end

app/workers/thread_resolve_worker.rb

@@ -6,9 +6,9 @@ class ThreadResolveWorker
 
   sidekiq_options queue: 'pull', retry: 3
 
-  def perform(child_status_id, parent_url)
+  def perform(child_status_id, parent_url, options = {})
     child_status  = Status.find(child_status_id)
-    parent_status = FetchRemoteStatusService.new.call(parent_url)
+    parent_status = FetchRemoteStatusService.new.call(parent_url, **options.deep_symbolize_keys)
 
     return if parent_status.nil?
 

app/workers/unfollow_follow_worker.rb

@@ -10,12 +10,7 @@ class UnfollowFollowWorker
     old_target_account = Account.find(old_target_account_id)
     new_target_account = Account.find(new_target_account_id)
 
-    follow  = follower_account.active_relationships.find_by(target_account: old_target_account)
-    reblogs = follow&.show_reblogs?
-    notify  = follow&.notify?
-
-    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, bypass_locked: bypass_locked, bypass_limit: true)
-    UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true)
+    FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked)
   rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
     true
   end

bin/tootctl

@@ -5,7 +5,9 @@ require_relative '../config/boot'
 require_relative '../lib/cli'
 
 begin
-  Mastodon::CLI.start(ARGV)
+  Chewy.strategy(:mastodon) do
+    Mastodon::CLI.start(ARGV)
+  end
 rescue Interrupt
   exit(130)
 end

chart/values.yaml

@@ -8,7 +8,7 @@ image:
   # built from the most recent commit
   #
   # tag: latest
-  tag: v3.5.2
+  tag: v3.5.5
   # use `Always` when using `latest` tag
   pullPolicy: IfNotPresent
 

config/database.yml

@@ -4,6 +4,7 @@ default: &default
   timeout: 5000
   encoding: unicode
   sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %>
+  application_name: ''
 
 development:
   <<: *default

config/initializers/chewy.rb

@@ -19,7 +19,6 @@ Chewy.settings = {
 # cycle, which takes care of checking if Elasticsearch is enabled
 # or not. However, mind that for the Rails console, the :urgent
 # strategy is set automatically with no way to override it.
-Chewy.root_strategy              = :mastodon
 Chewy.request_strategy           = :mastodon
 Chewy.use_after_commit_callbacks = false
 

config/initializers/content_security_policy.rb

@@ -26,6 +26,7 @@ Rails.application.config.content_security_policy do |p|
   p.media_src       :self, :https, :data, assets_host
   p.frame_src       :self, :https
   p.manifest_src    :self, assets_host
+  p.form_action     :self
 
   if Rails.env.development?
     webpacker_urls = %w(ws http).map { |protocol| "#{protocol}#{Webpacker.dev_server.https? ? 's' : ''}://#{Webpacker.dev_server.host_with_port}" }

config/initializers/rack_attack.rb

@@ -17,6 +17,18 @@ class Rack::Attack
       @remote_ip ||= (@env["action_dispatch.remote_ip"] || ip).to_s
     end
 
+    def throttleable_remote_ip
+      @throttleable_remote_ip ||= begin
+        ip = IPAddr.new(remote_ip)
+
+        if ip.ipv6?
+          ip.mask(64)
+        else
+          ip
+        end
+      end.to_s
+    end
+
     def authenticated_user_id
       authenticated_token&.resource_owner_id
     end
@@ -29,6 +41,10 @@ class Rack::Attack
       path.start_with?('/api')
     end
 
+    def path_matches?(other_path)
+      /\A#{Regexp.escape(other_path)}(\..*)?\z/ =~ path
+    end
+
     def web_request?
       !api_request?
     end
@@ -51,19 +67,19 @@ class Rack::Attack
   end
 
   throttle('throttle_unauthenticated_api', limit: 300, period: 5.minutes) do |req|
-    req.remote_ip if req.api_request? && req.unauthenticated?
+    req.throttleable_remote_ip if req.api_request? && req.unauthenticated?
   end
 
   throttle('throttle_api_media', limit: 30, period: 30.minutes) do |req|
-    req.authenticated_user_id if req.post? && req.path.match?('^/api/v\d+/media')
+    req.authenticated_user_id if req.post? && req.path.match?(/\A\/api\/v\d+\/media\z/i)
   end
 
   throttle('throttle_media_proxy', limit: 30, period: 10.minutes) do |req|
-    req.remote_ip if req.path.start_with?('/media_proxy')
+    req.throttleable_remote_ip if req.path.start_with?('/media_proxy')
   end
 
   throttle('throttle_api_sign_up', limit: 5, period: 30.minutes) do |req|
-    req.remote_ip if req.post? && req.path == '/api/v1/accounts'
+    req.throttleable_remote_ip if req.post? && req.path == '/api/v1/accounts'
   end
 
   throttle('throttle_authenticated_paging', limit: 300, period: 15.minutes) do |req|
@@ -71,39 +87,34 @@ class Rack::Attack
   end
 
   throttle('throttle_unauthenticated_paging', limit: 300, period: 15.minutes) do |req|
-    req.remote_ip if req.paging_request? && req.unauthenticated?
+    req.throttleable_remote_ip if req.paging_request? && req.unauthenticated?
   end
 
-  API_DELETE_REBLOG_REGEX = /\A\/api\/v1\/statuses\/[\d]+\/unreblog/.freeze
-  API_DELETE_STATUS_REGEX = /\A\/api\/v1\/statuses\/[\d]+/.freeze
+  API_DELETE_REBLOG_REGEX = /\A\/api\/v1\/statuses\/[\d]+\/unreblog\z/.freeze
+  API_DELETE_STATUS_REGEX = /\A\/api\/v1\/statuses\/[\d]+\z/.freeze
 
   throttle('throttle_api_delete', limit: 30, period: 30.minutes) do |req|
     req.authenticated_user_id if (req.post? && req.path.match?(API_DELETE_REBLOG_REGEX)) || (req.delete? && req.path.match?(API_DELETE_STATUS_REGEX))
   end
 
   throttle('throttle_sign_up_attempts/ip', limit: 25, period: 5.minutes) do |req|
-    if req.post? && req.path == '/auth'
-      addr = req.remote_ip
-      addr = IPAddr.new(addr) if addr.is_a?(String)
-      addr = addr.mask(64) if addr.ipv6?
-      addr.to_s
-    end
+    req.throttleable_remote_ip if req.post? && req.path_matches?('/auth')
   end
 
   throttle('throttle_password_resets/ip', limit: 25, period: 5.minutes) do |req|
-    req.remote_ip if req.post? && req.path == '/auth/password'
+    req.throttleable_remote_ip if req.post? && req.path_matches?('/auth/password')
   end
 
   throttle('throttle_password_resets/email', limit: 5, period: 30.minutes) do |req|
-    req.params.dig('user', 'email').presence if req.post? && req.path == '/auth/password'
+    req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/password')
   end
 
   throttle('throttle_email_confirmations/ip', limit: 25, period: 5.minutes) do |req|
-    req.remote_ip if req.post? && %w(/auth/confirmation /api/v1/emails/confirmations).include?(req.path)
+    req.throttleable_remote_ip if req.post? && (req.path_matches?('/auth/confirmation') || req.path == '/api/v1/emails/confirmations')
   end
 
   throttle('throttle_email_confirmations/email', limit: 5, period: 30.minutes) do |req|
-    if req.post? && req.path == '/auth/password'
+    if req.post? && req.path_matches?('/auth/password')
       req.params.dig('user', 'email').presence
     elsif req.post? && req.path == '/api/v1/emails/confirmations'
       req.authenticated_user_id
@@ -111,11 +122,11 @@ class Rack::Attack
   end
 
   throttle('throttle_login_attempts/ip', limit: 25, period: 5.minutes) do |req|
-    req.remote_ip if req.post? && req.path == '/auth/sign_in'
+    req.throttleable_remote_ip if req.post? && req.path_matches?('/auth/sign_in')
   end
 
   throttle('throttle_login_attempts/email', limit: 25, period: 1.hour) do |req|
-    req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path == '/auth/sign_in'
+    req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in')
   end
 
   self.throttled_responder = lambda do |request|

config/locales/en.yml

@@ -783,6 +783,12 @@ en:
         message_html: You haven't defined any server rules.
       sidekiq_process_check:
         message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
+      upload_check_privacy_error:
+        action: Check here for more information
+        message_html: "<strong>Your web server is misconfigured. The privacy of your users is at risk.</strong>"
+      upload_check_privacy_error_object_storage:
+        action: Check here for more information
+        message_html: "<strong>Your object storage is misconfigured. The privacy of your users is at risk.</strong>"
     tags:
       review: Review status
       updated_msg: Hashtag settings updated successfully
@@ -1323,6 +1329,7 @@ en:
   relationships:
     activity: Account activity
     dormant: Dormant
+    follow_failure: Could not follow some of the selected accounts.
     follow_selected_followers: Follow selected followers
     followers: Followers
     following: Following

config/routes.rb

@@ -47,7 +47,7 @@ Rails.application.routes.draw do
     end
   end
 
-  devise_for :users, path: 'auth', controllers: {
+  devise_for :users, path: 'auth', format: false, controllers: {
     omniauth_callbacks: 'auth/omniauth_callbacks',
     sessions:           'auth/sessions',
     registrations:      'auth/registrations',
@@ -182,7 +182,8 @@ Rails.application.routes.draw do
   resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update]
 
   get '/public', to: 'public_timelines#show', as: :public_timeline
-  get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy
+  get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false
+  get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false
 
   resource :authorize_interaction, only: [:show, :create]
   resource :share, only: [:show, :create]
@@ -353,7 +354,7 @@ Rails.application.routes.draw do
 
   get '/admin', to: redirect('/admin/dashboard', status: 302)
 
-  namespace :api do
+  namespace :api, format: false do
     # OEmbed
     get '/oembed', to: 'oembed#show', as: :oembed
 
@@ -394,7 +395,9 @@ Rails.application.routes.draw do
         resources :list, only: :show
       end
 
-      resources :streaming, only: [:index]
+      get '/streaming', to: 'streaming#index'
+      get '/streaming/(*any)', to: 'streaming#index'
+
       resources :custom_emojis, only: [:index]
       resources :suggestions, only: [:index, :destroy]
       resources :scheduled_statuses, only: [:index, :show, :update, :destroy]

docker-compose.yml

@@ -44,7 +44,7 @@ services:
 
   web:
     build: .
-    image: tootsuite/mastodon
+    image: ghcr.io/mastodon/mastodon:v3.5.7
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -65,7 +65,7 @@ services:
 
   streaming:
     build: .
-    image: tootsuite/mastodon
+    image: ghcr.io/mastodon/mastodon:v3.5.7
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -83,7 +83,7 @@ services:
 
   sidekiq:
     build: .
-    image: tootsuite/mastodon
+    image: ghcr.io/mastodon/mastodon:v3.5.7
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/mastodon/sidekiq_middleware.rb

@@ -3,8 +3,8 @@
 class Mastodon::SidekiqMiddleware
   BACKTRACE_LIMIT = 3
 
-  def call(*)
-    yield
+  def call(*, &block)
+    Chewy.strategy(:mastodon, &block)
   rescue Mastodon::HostValidationError
     # Do not retry
   rescue => e

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      3
+      7
     end
 
     def flags

lib/paperclip/blurhash_transcoder.rb

@@ -5,7 +5,7 @@ module Paperclip
     def make
       return @file unless options[:style] == :small || options[:blurhash]
 
-      pixels   = convert(':source RGB:-', source: File.expand_path(@file.path)).unpack('C*')
+      pixels   = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*')
       geometry = options.fetch(:file_geometry_parser).from_file(@file)
 
       attachment.instance.blurhash = Blurhash.encode(geometry.width, geometry.height, pixels, **(options[:blurhash] || {}))

lib/sanitize_ext/sanitize_config.rb

@@ -49,7 +49,7 @@ class Sanitize
         end
       end
 
-      current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme)
+      current_node.replace(Nokogiri::XML::Text.new(current_node.text, current_node.document)) unless LINK_PROTOCOLS.include?(scheme)
     end
 
     UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env|

spec/controllers/admin/accounts_controller_spec.rb

@@ -147,6 +147,87 @@ RSpec.describe Admin::AccountsController, type: :controller do
     end
   end
 
+  describe 'POST #approve' do
+    subject { post :approve, params: { id: account.id } }
+
+    let(:current_user) { Fabricate(:user, role: role) }
+    let(:account) { user.account }
+    let(:user) { Fabricate(:user) }
+
+    before do
+      account.user.update(approved: false)
+    end
+
+    context 'when user is admin' do
+      let(:role) { 'admin' }
+
+      it 'succeeds in approving account' do
+        is_expected.to redirect_to admin_accounts_path(status: 'pending')
+        expect(user.reload).to be_approved
+      end
+
+      it 'logs action' do
+        is_expected.to have_http_status :found
+
+        log_item = Admin::ActionLog.last
+
+        expect(log_item).to_not be_nil
+        expect(log_item.action).to eq :approve
+        expect(log_item.account_id).to eq current_user.account_id
+        expect(log_item.target_id).to eq account.user.id
+      end
+    end
+
+    context 'when user is not admin' do
+      let(:role) { 'user' }
+
+      it 'fails to approve account' do
+        is_expected.to have_http_status :forbidden
+        expect(user.reload).not_to be_approved
+      end
+    end
+  end
+
+  describe 'POST #reject' do
+    subject { post :reject, params: { id: account.id } }
+
+    let(:current_user) { Fabricate(:user, role: role) }
+    let(:account) { user.account }
+    let(:user) { Fabricate(:user) }
+
+    before do
+      account.user.update(approved: false)
+    end
+
+    context 'when user is admin' do
+      let(:role) { 'admin' }
+
+      it 'succeeds in rejecting account' do
+        is_expected.to redirect_to admin_accounts_path(status: 'pending')
+      end
+
+      it 'logs action' do
+        is_expected.to have_http_status :found
+
+        log_item = Admin::ActionLog.last
+
+        expect(log_item).to_not be_nil
+        expect(log_item.action).to eq :reject
+        expect(log_item.account_id).to eq current_user.account_id
+        expect(log_item.target_id).to eq account.user.id
+      end
+    end
+
+    context 'when user is not admin' do
+      let(:role) { 'user' }
+
+      it 'fails to reject account' do
+        is_expected.to have_http_status :forbidden
+        expect(user.reload).not_to be_approved
+      end
+    end
+  end
+
   describe 'POST #redownload' do
     subject { post :redownload, params: { id: account.id } }
 

spec/controllers/admin/domain_blocks_controller_spec.rb

@@ -49,6 +49,53 @@ RSpec.describe Admin::DomainBlocksController, type: :controller do
     end
   end
 
+  describe 'PUT #update' do
+    let!(:remote_account) { Fabricate(:account, domain: 'example.com') }
+    let(:domain_block)    { Fabricate(:domain_block, domain: 'example.com', severity: original_severity) }
+
+    before do
+      BlockDomainService.new.call(domain_block)
+    end
+
+    let(:subject) do
+      post :update, params: { id: domain_block.id, domain_block: { domain: 'example.com', severity: new_severity } }
+    end
+
+    context 'downgrading a domain suspension to silence' do
+      let(:original_severity) { 'suspend' }
+      let(:new_severity)      { 'silence' }
+
+      it 'changes the block severity' do
+        expect { subject }.to change { domain_block.reload.severity }.from('suspend').to('silence')
+      end
+
+      it 'undoes individual suspensions' do
+        expect { subject }.to change { remote_account.reload.suspended? }.from(true).to(false)
+      end
+
+      it 'performs individual silences' do
+        expect { subject }.to change { remote_account.reload.silenced? }.from(false).to(true)
+      end
+    end
+
+    context 'upgrading a domain silence to suspend' do
+      let(:original_severity) { 'silence' }
+      let(:new_severity)      { 'suspend' }
+
+      it 'changes the block severity' do
+        expect { subject }.to change { domain_block.reload.severity }.from('silence').to('suspend')
+      end
+
+      it 'undoes individual silences' do
+        expect { subject }.to change { remote_account.reload.silenced? }.from(true).to(false)
+      end
+
+      it 'performs individual suspends' do
+        expect { subject }.to change { remote_account.reload.suspended? }.from(false).to(true)
+      end
+    end
+  end
+
   describe 'DELETE #destroy' do
     it 'unblocks the domain' do
       service = double(call: true)

spec/controllers/admin/reports/actions_controller_spec.rb

@@ -0,0 +1,42 @@
+require 'rails_helper'
+
+describe Admin::Reports::ActionsController do
+  render_views
+
+  let(:user) { Fabricate(:user, role: 'moderator') }
+  let(:account) { Fabricate(:account) }
+  let!(:status) { Fabricate(:status, account: account) }
+  let(:media_attached_status) { Fabricate(:status, account: account) }
+  let!(:media_attachment) { Fabricate(:media_attachment, account: account, status: media_attached_status) }
+  let(:media_attached_deleted_status) { Fabricate(:status, account: account, deleted_at: 1.day.ago) }
+  let!(:media_attachment2) { Fabricate(:media_attachment, account: account, status: media_attached_deleted_status) }
+  let(:last_media_attached_status) { Fabricate(:status, account: account) }
+  let!(:last_media_attachment) { Fabricate(:media_attachment, account: account, status: last_media_attached_status) }
+  let!(:last_status) { Fabricate(:status, account: account) }
+
+  before do
+    sign_in user, scope: :user
+  end
+
+  describe 'POST #create' do
+    let(:report) { Fabricate(:report, status_ids: status_ids, account: user.account, target_account: account) }
+    let(:status_ids) { [media_attached_status.id, media_attached_deleted_status.id] }
+
+    before do
+      post :create, params: { report_id: report.id, action => '' }
+    end
+
+    context 'when action is mark_as_sensitive' do
+
+      let(:action) { 'mark_as_sensitive' }
+
+      it 'resolves the report' do
+        expect(report.reload.action_taken_at).to_not be_nil
+      end
+
+      it 'marks the non-deleted as sensitive' do
+        expect(media_attached_status.reload.sensitive).to eq true
+      end
+    end
+  end
+end

spec/controllers/api/v1/admin/accounts_controller_spec.rb

@@ -100,6 +100,15 @@ RSpec.describe Api::V1::Admin::AccountsController, type: :controller do
     it 'approves user' do
       expect(account.reload.user_approved?).to be true
     end
+
+    it 'logs action' do
+      log_item = Admin::ActionLog.last
+
+      expect(log_item).to_not be_nil
+      expect(log_item.action).to eq :approve
+      expect(log_item.account_id).to eq user.account_id
+      expect(log_item.target_id).to eq account.user.id
+    end
   end
 
   describe 'POST #reject' do
@@ -118,6 +127,15 @@ RSpec.describe Api::V1::Admin::AccountsController, type: :controller do
     it 'removes user' do
       expect(User.where(id: account.user.id).count).to eq 0
     end
+
+    it 'logs action' do
+      log_item = Admin::ActionLog.last
+
+      expect(log_item).to_not be_nil
+      expect(log_item.action).to eq :reject
+      expect(log_item.account_id).to eq user.account_id
+      expect(log_item.target_id).to eq account.user.id
+    end
   end
 
   describe 'POST #enable' do

spec/controllers/relationships_controller_spec.rb

@@ -55,7 +55,7 @@ describe RelationshipsController do
     end
 
     context 'when select parameter is provided' do
-      subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, block_domains: '' } }
+      subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, remove_domains_from_followers: '' } }
 
       it 'soft-blocks followers from selected domains' do
         poopfeast.follow!(user.account)
@@ -66,6 +66,15 @@ describe RelationshipsController do
         expect(poopfeast.following?(user.account)).to be false
       end
 
+      it 'does not unfollow users from selected domains' do
+        user.account.follow!(poopfeast)
+
+        sign_in user, scope: :user
+        subject
+
+        expect(user.account.following?(poopfeast)).to be true
+      end
+
       include_examples 'authenticate user'
       include_examples 'redirects back to followers page'
     end

spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb

@@ -248,7 +248,7 @@ describe Settings::TwoFactorAuthentication::WebauthnCredentialsController do
 
               post :create, params: { credential: new_webauthn_credential, nickname: 'USB Key' }
 
-              expect(response).to have_http_status(500)
+              expect(response).to have_http_status(422)
               expect(flash[:error]).to be_present
             end
           end
@@ -268,7 +268,7 @@ describe Settings::TwoFactorAuthentication::WebauthnCredentialsController do
 
               post :create, params: { credential: new_webauthn_credential, nickname: nickname }
 
-              expect(response).to have_http_status(500)
+              expect(response).to have_http_status(422)
               expect(flash[:error]).to be_present
             end
           end

spec/lib/activitypub/activity/add_spec.rb

@@ -48,7 +48,7 @@ RSpec.describe ActivityPub::Activity::Add do
         end
 
         it 'fetches the status and pins it' do
-          allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil|
+          allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil, request_id: nil|
             expect(uri).to eq 'https://example.com/unknown'
             expect(id).to eq true
             expect(on_behalf_of&.following?(sender)).to eq true
@@ -62,7 +62,7 @@ RSpec.describe ActivityPub::Activity::Add do
 
       context 'when there is no local follower' do
         it 'tries to fetch the status' do
-          allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil|
+          allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil, request_id: nil|
             expect(uri).to eq 'https://example.com/unknown'
             expect(id).to eq true
             expect(on_behalf_of).to eq nil

spec/lib/activitypub/activity/flag_spec.rb

@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 RSpec.describe ActivityPub::Activity::Flag do
-  let(:sender)  { Fabricate(:account, domain: 'example.com', uri: 'http://example.com/account') }
+  let(:sender)  { Fabricate(:account, username: 'example.com', domain: 'example.com', uri: 'http://example.com/actor') }
   let(:flagged) { Fabricate(:account) }
   let(:status)  { Fabricate(:status, account: flagged, uri: 'foobar') }
   let(:flag_id) { nil }
@@ -23,16 +23,88 @@ RSpec.describe ActivityPub::Activity::Flag do
   describe '#perform' do
     subject { described_class.new(json, sender) }
 
-    before do
-      subject.perform
+    context 'when the reported status is public' do
+      before do
+        subject.perform
+      end
+
+      it 'creates a report' do
+        report = Report.find_by(account: sender, target_account: flagged)
+
+        expect(report).to_not be_nil
+        expect(report.comment).to eq 'Boo!!'
+        expect(report.status_ids).to eq [status.id]
+      end
     end
 
-    it 'creates a report' do
-      report = Report.find_by(account: sender, target_account: flagged)
+    context 'when the reported status is private and should not be visible to the remote server' do
+      let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
 
-      expect(report).to_not be_nil
-      expect(report.comment).to eq 'Boo!!'
-      expect(report.status_ids).to eq [status.id]
+      before do
+        subject.perform
+      end
+
+      it 'creates a report with no attached status' do
+        report = Report.find_by(account: sender, target_account: flagged)
+
+        expect(report).to_not be_nil
+        expect(report.comment).to eq 'Boo!!'
+        expect(report.status_ids).to eq []
+      end
+    end
+
+    context 'when the reported status is private and the author has a follower on the remote instance' do
+      let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
+      let(:follower) { Fabricate(:account, domain: 'example.com', uri: 'http://example.com/users/account') }
+
+      before do
+        follower.follow!(flagged)
+        subject.perform
+      end
+
+      it 'creates a report with the attached status' do
+        report = Report.find_by(account: sender, target_account: flagged)
+
+        expect(report).to_not be_nil
+        expect(report.comment).to eq 'Boo!!'
+        expect(report.status_ids).to eq [status.id]
+      end
+    end
+
+    context 'when the reported status is private and the author mentions someone else on the remote instance' do
+      let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
+      let(:mentioned) { Fabricate(:account, domain: 'example.com', uri: 'http://example.com/users/account') }
+
+      before do
+        status.mentions.create(account: mentioned)
+        subject.perform
+      end
+
+      it 'creates a report with the attached status' do
+        report = Report.find_by(account: sender, target_account: flagged)
+
+        expect(report).to_not be_nil
+        expect(report.comment).to eq 'Boo!!'
+        expect(report.status_ids).to eq [status.id]
+      end
+    end
+
+    context 'when the reported status is private and the author mentions someone else on the local instance' do
+      let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
+      let(:mentioned) { Fabricate(:account) }
+
+      before do
+        status.mentions.create(account: mentioned)
+        subject.perform
+      end
+
+      it 'creates a report with no attached status' do
+        report = Report.find_by(account: sender, target_account: flagged)
+
+        expect(report).to_not be_nil
+        expect(report.comment).to eq 'Boo!!'
+        expect(report.status_ids).to eq []
+      end
     end
   end
 

spec/lib/sanitize_config_spec.rb

@@ -38,6 +38,10 @@ describe Sanitize::Config do
       expect(Sanitize.fragment('<a href="foo://bar">Test</a>', subject)).to eq 'Test'
     end
 
+    it 'does not re-interpret HTML when removing unsupported links' do
+      expect(Sanitize.fragment('<a href="foo://bar">Test&lt;a href="https://example.com"&gt;test&lt;/a&gt;</a>', subject)).to eq 'Test&lt;a href="https://example.com"&gt;test&lt;/a&gt;'
+    end
+
     it 'keeps a with href' do
       expect(Sanitize.fragment('<a href="http://example.com">Test</a>', subject)).to eq '<a href="http://example.com" rel="nofollow noopener noreferrer" target="_blank">Test</a>'
     end

spec/services/activitypub/fetch_remote_status_service_spec.rb

@@ -223,4 +223,98 @@ RSpec.describe ActivityPub::FetchRemoteStatusService, type: :service do
       end
     end
   end
+
+  context 'statuses referencing other statuses' do
+    before do
+      stub_const 'ActivityPub::FetchRemoteStatusService::DISCOVERIES_PER_REQUEST', 5
+    end
+
+    context 'using inReplyTo' do
+      let(:object) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: "https://foo.bar/@foo/1",
+          type: 'Note',
+          content: 'Lorem ipsum',
+          inReplyTo: 'https://foo.bar/@foo/2',
+          attributedTo: ActivityPub::TagManager.instance.uri_for(sender),
+        }
+      end
+
+      before do
+        8.times do |i|
+          status_json = {
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            id: "https://foo.bar/@foo/#{i}",
+            type: 'Note',
+            content: 'Lorem ipsum',
+            inReplyTo: "https://foo.bar/@foo/#{i + 1}",
+            attributedTo: ActivityPub::TagManager.instance.uri_for(sender),
+            to: 'as:Public',
+          }.with_indifferent_access
+          stub_request(:get, "https://foo.bar/@foo/#{i}").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
+        end
+      end
+
+      it 'creates at least some statuses' do
+        expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_least(2)
+      end
+
+      it 'creates no more account than the limit allows' do
+        expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_most(5)
+      end
+    end
+
+    context 'using replies' do
+      let(:object) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: "https://foo.bar/@foo/1",
+          type: 'Note',
+          content: 'Lorem ipsum',
+          replies: {
+            type: 'Collection',
+            id: 'https://foo.bar/@foo/1/replies',
+            first: {
+              type: 'CollectionPage',
+              partOf: 'https://foo.bar/@foo/1/replies',
+              items: ['https://foo.bar/@foo/2'],
+            },
+          },
+          attributedTo: ActivityPub::TagManager.instance.uri_for(sender),
+        }
+      end
+
+      before do
+        8.times do |i|
+          status_json = {
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            id: "https://foo.bar/@foo/#{i}",
+            type: 'Note',
+            content: 'Lorem ipsum',
+            replies: {
+              type: 'Collection',
+              id: "https://foo.bar/@foo/#{i}/replies",
+              first: {
+                type: 'CollectionPage',
+                partOf: "https://foo.bar/@foo/#{i}/replies",
+                items: ["https://foo.bar/@foo/#{i+1}"],
+              },
+            },
+            attributedTo: ActivityPub::TagManager.instance.uri_for(sender),
+            to: 'as:Public',
+          }.with_indifferent_access
+          stub_request(:get, "https://foo.bar/@foo/#{i}").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
+        end
+      end
+
+      it 'creates at least some statuses' do
+        expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_least(2)
+      end
+
+      it 'creates no more account than the limit allows' do
+        expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_most(5)
+      end
+    end
+  end
 end

spec/services/activitypub/process_account_service_spec.rb

@@ -109,4 +109,98 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
       end
     end
   end
+
+  context 'discovering many subdomains in a short timeframe' do
+    before do
+      stub_const 'ActivityPub::ProcessAccountService::SUBDOMAINS_RATELIMIT', 5
+    end
+
+    let(:subject) do
+      8.times do |i|
+        domain = "test#{i}.testdomain.com"
+        json = {
+          id: "https://#{domain}/users/1",
+          type: 'Actor',
+          inbox: "https://#{domain}/inbox",
+        }.with_indifferent_access
+        described_class.new.call('alice', domain, json)
+      end
+    end
+
+    it 'creates at least some accounts' do
+      expect { subject }.to change { Account.remote.count }.by_at_least(2)
+    end
+
+    it 'creates no more account than the limit allows' do
+      expect { subject }.to change { Account.remote.count }.by_at_most(5)
+    end
+  end
+
+  context 'accounts referencing other accounts' do
+    before do
+      stub_const 'ActivityPub::ProcessAccountService::DISCOVERIES_PER_REQUEST', 5
+    end
+
+    let(:payload) do
+      {
+        '@context': ['https://www.w3.org/ns/activitystreams'],
+        id: 'https://foo.test/users/1',
+        type: 'Person',
+        inbox: 'https://foo.test/inbox',
+        featured: 'https://foo.test/users/1/featured',
+        preferredUsername: 'user1',
+      }.with_indifferent_access
+    end
+
+    before do
+      8.times do |i|
+        actor_json = {
+          '@context': ['https://www.w3.org/ns/activitystreams'],
+          id: "https://foo.test/users/#{i}",
+          type: 'Person',
+          inbox: 'https://foo.test/inbox',
+          featured: "https://foo.test/users/#{i}/featured",
+          preferredUsername: "user#{i}",
+        }.with_indifferent_access
+        status_json = {
+          '@context': ['https://www.w3.org/ns/activitystreams'],
+          id: "https://foo.test/users/#{i}/status",
+          attributedTo: "https://foo.test/users/#{i}",
+          type: 'Note',
+          content: "@user#{i + 1} test",
+          tag: [
+            {
+              type: 'Mention',
+              href: "https://foo.test/users/#{i + 1}",
+              name: "@user#{i + 1 }",
+            }
+          ],
+          to: [ 'as:Public', "https://foo.test/users/#{i + 1}" ]
+        }.with_indifferent_access
+        featured_json = {
+          '@context': ['https://www.w3.org/ns/activitystreams'],
+          id: "https://foo.test/users/#{i}/featured",
+          type: 'OrderedCollection',
+          totelItems: 1,
+          orderedItems: [status_json],
+        }.with_indifferent_access
+        webfinger = {
+          subject: "acct:user#{i}@foo.test",
+          links: [{ rel: 'self', href: "https://foo.test/users/#{i}" }],
+        }.with_indifferent_access
+        stub_request(:get, "https://foo.test/users/#{i}").to_return(status: 200, body: actor_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, "https://foo.test/users/#{i}/featured").to_return(status: 200, body: featured_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, "https://foo.test/users/#{i}/status").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, "https://foo.test/.well-known/webfinger?resource=acct:user#{i}@foo.test").to_return(body: webfinger.to_json, headers: { 'Content-Type': 'application/jrd+json' })
+      end
+    end
+
+    it 'creates at least some accounts' do
+      expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_least(2)
+    end
+
+    it 'creates no more account than the limit allows' do
+      expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_most(5)
+    end
+  end
 end

spec/services/activitypub/process_status_update_service_spec.rb

@@ -331,7 +331,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
 
     context 'originally without media attachments' do
       before do
-        allow(RedownloadMediaWorker).to receive(:perform_async)
+        stub_request(:get, 'https://example.com/foo.png').to_return(body: attachment_fixture('emojo.png'))
         subject.call(status, json)
       end
 
@@ -355,8 +355,8 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
         expect(media_attachment.remote_url).to eq 'https://example.com/foo.png'
       end
 
-      it 'queues download of media attachments' do
-        expect(RedownloadMediaWorker).to have_received(:perform_async)
+      it 'fetches the attachment' do
+        expect(a_request(:get, 'https://example.com/foo.png')).to have_been_made
       end
 
       it 'records media change in edit' do

spec/services/fetch_remote_status_service_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe FetchRemoteStatusService, type: :service do
   end
 
   context 'protocol is :activitypub' do
-    subject { described_class.new.call(note[:id], prefetched_body) }
+    subject { described_class.new.call(note[:id], prefetched_body: prefetched_body) }
     let(:prefetched_body) { Oj.dump(note) }
 
     before do

spec/services/report_service_spec.rb

@@ -28,6 +28,31 @@ RSpec.describe ReportService, type: :service do
     end
   end
 
+  context 'when the reported status is a DM' do
+    let(:target_account) { Fabricate(:account) }
+    let(:status) { Fabricate(:status, account: target_account, visibility: :direct) }
+
+    subject do
+      -> { described_class.new.call(source_account, target_account, status_ids: [status.id]) }
+    end
+
+    context 'when it is addressed to the reporter' do
+      before do
+        status.mentions.create(account: source_account)
+      end
+
+      it 'creates a report' do
+        is_expected.to change { target_account.targeted_reports.count }.from(0).to(1)
+      end
+    end
+
+    context 'when it is not addressed to the reporter' do
+      it 'errors out' do
+        is_expected.to raise_error
+      end
+    end
+  end
+
   context 'when other reports already exist for the same target' do
     let!(:target_account) { Fabricate(:account) }
     let!(:other_report)   { Fabricate(:report, target_account: target_account) }

spec/services/suspend_account_service_spec.rb

@@ -13,6 +13,8 @@ RSpec.describe SuspendAccountService, type: :service do
 
       local_follower.follow!(account)
       list.accounts << account
+
+      account.suspend!
     end
 
     it "unmerges from local followers' feeds" do
@@ -21,8 +23,8 @@ RSpec.describe SuspendAccountService, type: :service do
       expect(FeedManager.instance).to have_received(:unmerge_from_list).with(account, list)
     end
 
-    it 'marks account as suspended' do
-      expect { subject }.to change { account.suspended? }.from(false).to(true)
+    it 'does not change the “suspended” flag' do
+      expect { subject }.to_not change { account.suspended? }
     end
   end
 

spec/services/unsuspend_account_service_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe UnsuspendAccountService, type: :service do
       local_follower.follow!(account)
       list.accounts << account
 
-      account.suspend!(origin: :local)
+      account.unsuspend!
     end
   end
 
@@ -30,8 +30,8 @@ RSpec.describe UnsuspendAccountService, type: :service do
       stub_request(:post, 'https://bob.com/inbox').to_return(status: 201)
     end
 
-    it 'marks account as unsuspended' do
-      expect { subject }.to change { account.suspended? }.from(true).to(false)
+    it 'does not change the “suspended” flag' do
+      expect { subject }.to_not change { account.suspended? }
     end
 
     include_examples 'common behavior' do
@@ -83,8 +83,8 @@ RSpec.describe UnsuspendAccountService, type: :service do
           expect(FeedManager.instance).to have_received(:merge_into_list).with(account, list)
         end
 
-        it 'marks account as unsuspended' do
-          expect { subject }.to change { account.suspended? }.from(true).to(false)
+        it 'does not change the “suspended” flag' do
+          expect { subject }.to_not change { account.suspended? }
         end
       end
 
@@ -107,8 +107,8 @@ RSpec.describe UnsuspendAccountService, type: :service do
           expect(FeedManager.instance).to_not have_received(:merge_into_list).with(account, list)
         end
 
-        it 'does not mark the account as unsuspended' do
-          expect { subject }.not_to change { account.suspended? }
+        it 'marks account as suspended' do
+          expect { subject }.to change { account.suspended? }.from(false).to(true)
         end
       end
 

spec/workers/scheduler/user_cleanup_scheduler_spec.rb

@@ -0,0 +1,39 @@
+require 'rails_helper'
+
+describe Scheduler::UserCleanupScheduler do
+  subject { described_class.new }
+
+  let!(:new_unconfirmed_user) { Fabricate(:user) }
+  let!(:old_unconfirmed_user) { Fabricate(:user) }
+  let!(:confirmed_user)       { Fabricate(:user) }
+  let!(:moderation_note)      { Fabricate(:account_moderation_note, account: Fabricate(:account), target_account: old_unconfirmed_user.account) }
+
+  describe '#perform' do
+    before do
+      # Need to update the already-existing users because their initialization overrides confirmation_sent_at
+      new_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: Time.now.utc)
+      old_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: 1.week.ago)
+      confirmed_user.update!(confirmed_at: 1.day.ago)
+    end
+
+    it 'deletes the old unconfirmed user' do
+      expect { subject.perform }.to change { User.exists?(old_unconfirmed_user.id) }.from(true).to(false)
+    end
+
+    it "deletes the old unconfirmed user's account" do
+      expect { subject.perform }.to change { Account.exists?(old_unconfirmed_user.account_id) }.from(true).to(false)
+    end
+
+    it 'does not delete the new unconfirmed user or their account' do
+      subject.perform
+      expect(User.exists?(new_unconfirmed_user.id)).to be true
+      expect(Account.exists?(new_unconfirmed_user.account_id)).to be true
+    end
+
+    it 'does not delete the confirmed user or their account' do
+      subject.perform
+      expect(User.exists?(confirmed_user.id)).to be true
+      expect(Account.exists?(confirmed_user.account_id)).to be true
+    end
+  end
+end