From d8629e7b86b6fcaedc8b31de57dd95e85fe4fb04 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 21 Jul 2021 18:34:39 +0200
Subject: Add logging of S3-related errors (#16381)

---
 app/services/backup_service.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/services')

diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 6a1575616..037f519d3 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -167,7 +167,7 @@ class BackupService < BaseService
         io.write(buffer)
       end
     end
-  rescue Errno::ENOENT, Seahorse::Client::NetworkingError
-    Rails.logger.warn "Could not backup file #{filename}: file not found"
+  rescue Errno::ENOENT, Seahorse::Client::NetworkingError => e
+    Rails.logger.warn "Could not backup file #{filename}: #{e}"
   end
 end
-- 
cgit 


From 1d67acb72fa418a456c5e4762425702bfe7702b1 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Sat, 24 Jul 2021 14:41:46 +0200
Subject: Fix scoped order warning in RemoveStatusService (#16531)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes “Scoped order is ignored, it's forced to be batch order.”
---
 app/services/remove_status_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index b680c8e96..f9c3dcf78 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -102,7 +102,7 @@ class RemoveStatusService < BaseService
     # because once original status is gone, reblogs will disappear
     # without us being able to do all the fancy stuff
 
-    @status.reblogs.includes(:account).find_each do |reblog|
+    @status.reblogs.includes(:account).reorder(nil).find_each do |reblog|
       RemoveStatusService.new.call(reblog, original_removed: true)
     end
   end
-- 
cgit 


From 763ab0c7eb5430235ca8a354d11e00de1d8ba6dd Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Sun, 8 Aug 2021 15:29:57 +0200
Subject: Fix owned account notes not being deleted when an account is deleted
 (#16579)

* Add account_notes relationship

* Add tests

* Fix owned account notes not being deleted when an account is deleted

* Add post-migration to clean up orphaned account notes
---
 app/models/concerns/account_interactions.rb         |  3 +++
 app/services/delete_account_service.rb              |  2 ++
 .../20210808071221_clear_orphaned_account_notes.rb  | 21 +++++++++++++++++++++
 db/schema.rb                                        |  2 +-
 spec/services/delete_account_service_spec.rb        |  5 ++++-
 5 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 db/post_migrate/20210808071221_clear_orphaned_account_notes.rb

(limited to 'app/services')

diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index 958f6c78e..4bf62539c 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -81,6 +81,9 @@ module AccountInteractions
     has_many :following, -> { order('follows.id desc') }, through: :active_relationships,  source: :target_account
     has_many :followers, -> { order('follows.id desc') }, through: :passive_relationships, source: :account
 
+    # Account notes
+    has_many :account_notes, dependent: :destroy
+
     # Block relationships
     has_many :block_relationships, class_name: 'Block', foreign_key: 'account_id', dependent: :destroy
     has_many :blocking, -> { order('blocks.id desc') }, through: :block_relationships, source: :target_account
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index 182f0e127..d8270498a 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -4,6 +4,7 @@ class DeleteAccountService < BaseService
   include Payloadable
 
   ASSOCIATIONS_ON_SUSPEND = %w(
+    account_notes
     account_pins
     active_relationships
     aliases
@@ -34,6 +35,7 @@ class DeleteAccountService < BaseService
   # by foreign keys, making them safe to delete without loading
   # into memory
   ASSOCIATIONS_WITHOUT_SIDE_EFFECTS = %w(
+    account_notes
     account_pins
     aliases
     conversation_mutes
diff --git a/db/post_migrate/20210808071221_clear_orphaned_account_notes.rb b/db/post_migrate/20210808071221_clear_orphaned_account_notes.rb
new file mode 100644
index 000000000..71171658a
--- /dev/null
+++ b/db/post_migrate/20210808071221_clear_orphaned_account_notes.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class ClearOrphanedAccountNotes < ActiveRecord::Migration[5.2]
+  class Account < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+  end
+
+  class AccountNote < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account
+    belongs_to :target_account, class_name: 'Account'
+  end
+
+  def up
+    AccountNote.where('NOT EXISTS (SELECT * FROM users u WHERE u.account_id = account_notes.account_id)').in_batches.delete_all
+  end
+
+  def down
+    # nothing to do
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index b2929f693..a0a98eb03 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_06_30_000137) do
+ActiveRecord::Schema.define(version: 2021_08_08_071221) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
diff --git a/spec/services/delete_account_service_spec.rb b/spec/services/delete_account_service_spec.rb
index cd7d32d59..b1da97036 100644
--- a/spec/services/delete_account_service_spec.rb
+++ b/spec/services/delete_account_service_spec.rb
@@ -21,6 +21,8 @@ RSpec.describe DeleteAccountService, type: :service do
     let!(:favourite_notification) { Fabricate(:notification, account: local_follower, activity: favourite, type: :favourite) }
     let!(:follow_notification) { Fabricate(:notification, account: local_follower, activity: active_relationship, type: :follow) }
 
+    let!(:account_note) { Fabricate(:account_note, account: account) }
+
     subject do
       -> { described_class.new.call(account) }
     end
@@ -35,8 +37,9 @@ RSpec.describe DeleteAccountService, type: :service do
           account.active_relationships,
           account.passive_relationships,
           account.polls,
+          account.account_notes,
         ].map(&:count)
-      }.from([2, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+      }.from([2, 1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0, 0])
     end
 
     it 'deletes associated target records' do
-- 
cgit 


From 4ac78e2a066508a54de82f1d910ef2fd36c3d106 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 9 Aug 2021 23:11:50 +0200
Subject: Add feature to automatically delete old toots (#16529)

* Add account statuses cleanup policy model

* Record last inspected toot to delete to speed up successive calls to statuses_to_delete

* Add service to cleanup a given account's statuses within a budget

* Add worker to go through account policies and delete old toots

* Fix last inspected status id logic

All existing statuses older or equal to last inspected status id must be
kept by the current policy. This is an invariant that must be kept so that
resuming deletion from the last inspected status remains sound.

* Add tests

* Refactor scheduler and add tests

* Add user interface

* Add support for discriminating based on boosts/favs

* Add UI support for min_reblogs and min_favs, rework UI

* Address first round of review comments

* Replace Snowflake#id_at_start with with_random parameter

* Add tests

* Add tests for StatusesCleanupController

* Rework settings page

* Adjust load-avoiding mechanisms

* Please CodeClimate
---
 app/controllers/statuses_cleanup_controller.rb     |  35 ++
 app/models/account_statuses_cleanup_policy.rb      | 171 +++++++
 app/models/bookmark.rb                             |   8 +
 app/models/concerns/account_associations.rb        |   3 +
 app/models/favourite.rb                            |   7 +
 app/models/status_pin.rb                           |   8 +
 app/services/account_statuses_cleanup_service.rb   |  27 +
 app/views/statuses_cleanup/show.html.haml          |  45 ++
 .../accounts_statuses_cleanup_scheduler.rb         |  96 ++++
 config/locales/en.yml                              |  35 ++
 config/navigation.rb                               |   1 +
 config/routes.rb                                   |   1 +
 config/sidekiq.yml                                 |   4 +
 ...340_create_account_statuses_cleanup_policies.rb |  20 +
 db/schema.rb                                       |  18 +
 lib/mastodon/snowflake.rb                          |   7 +-
 .../statuses_cleanup_controller_spec.rb            |  27 +
 .../account_statuses_cleanup_policy_fabricator.rb  |   3 +
 .../models/account_statuses_cleanup_policy_spec.rb | 546 +++++++++++++++++++++
 .../account_statuses_cleanup_service_spec.rb       | 101 ++++
 .../accounts_statuses_cleanup_scheduler_spec.rb    | 127 +++++
 21 files changed, 1287 insertions(+), 3 deletions(-)
 create mode 100644 app/controllers/statuses_cleanup_controller.rb
 create mode 100644 app/models/account_statuses_cleanup_policy.rb
 create mode 100644 app/services/account_statuses_cleanup_service.rb
 create mode 100644 app/views/statuses_cleanup/show.html.haml
 create mode 100644 app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
 create mode 100644 db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb
 create mode 100644 spec/controllers/statuses_cleanup_controller_spec.rb
 create mode 100644 spec/fabricators/account_statuses_cleanup_policy_fabricator.rb
 create mode 100644 spec/models/account_statuses_cleanup_policy_spec.rb
 create mode 100644 spec/services/account_statuses_cleanup_service_spec.rb
 create mode 100644 spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/statuses_cleanup_controller.rb b/app/controllers/statuses_cleanup_controller.rb
new file mode 100644
index 000000000..be234cdcb
--- /dev/null
+++ b/app/controllers/statuses_cleanup_controller.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class StatusesCleanupController < ApplicationController
+  layout 'admin'
+
+  before_action :authenticate_user!
+  before_action :set_policy
+  before_action :set_body_classes
+
+  def show; end
+
+  def update
+    if @policy.update(resource_params)
+      redirect_to statuses_cleanup_path, notice: I18n.t('generic.changes_saved_msg')
+    else
+      render action: :show
+    end
+  rescue ActionController::ParameterMissing
+    # Do nothing
+  end
+
+  private
+
+  def set_policy
+    @policy = current_account.statuses_cleanup_policy || current_account.build_statuses_cleanup_policy(enabled: false)
+  end
+
+  def resource_params
+    params.require(:account_statuses_cleanup_policy).permit(:enabled, :min_status_age, :keep_direct, :keep_pinned, :keep_polls, :keep_media, :keep_self_fav, :keep_self_bookmark, :min_favs, :min_reblogs)
+  end
+
+  def set_body_classes
+    @body_classes = 'admin'
+  end
+end
diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb
new file mode 100644
index 000000000..705ccff54
--- /dev/null
+++ b/app/models/account_statuses_cleanup_policy.rb
@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_statuses_cleanup_policies
+#
+#  id                 :bigint           not null, primary key
+#  account_id         :bigint           not null
+#  enabled            :boolean          default(TRUE), not null
+#  min_status_age     :integer          default(1209600), not null
+#  keep_direct        :boolean          default(TRUE), not null
+#  keep_pinned        :boolean          default(TRUE), not null
+#  keep_polls         :boolean          default(FALSE), not null
+#  keep_media         :boolean          default(FALSE), not null
+#  keep_self_fav      :boolean          default(TRUE), not null
+#  keep_self_bookmark :boolean          default(TRUE), not null
+#  min_favs           :integer
+#  min_reblogs        :integer
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#
+class AccountStatusesCleanupPolicy < ApplicationRecord
+  include Redisable
+
+  ALLOWED_MIN_STATUS_AGE = [
+    2.weeks.seconds,
+    1.month.seconds,
+    2.months.seconds,
+    3.months.seconds,
+    6.months.seconds,
+    1.year.seconds,
+    2.years.seconds,
+  ].freeze
+
+  EXCEPTION_BOOLS      = %w(keep_direct keep_pinned keep_polls keep_media keep_self_fav keep_self_bookmark).freeze
+  EXCEPTION_THRESHOLDS = %w(min_favs min_reblogs).freeze
+
+  # Depending on the cleanup policy, the query to discover the next
+  # statuses to delete my get expensive if the account has a lot of old
+  # statuses otherwise excluded from deletion by the other exceptions.
+  #
+  # Therefore, `EARLY_SEARCH_CUTOFF` is meant to be the maximum number of
+  # old statuses to be considered for deletion prior to checking exceptions.
+  #
+  # This is used in `compute_cutoff_id` to provide a `max_id` to
+  # `statuses_to_delete`.
+  EARLY_SEARCH_CUTOFF = 5_000
+
+  belongs_to :account
+
+  validates :min_status_age, inclusion: { in: ALLOWED_MIN_STATUS_AGE }
+  validates :min_favs, numericality: { greater_than_or_equal_to: 1, allow_nil: true }
+  validates :min_reblogs, numericality: { greater_than_or_equal_to: 1, allow_nil: true }
+  validate :validate_local_account
+
+  before_save :update_last_inspected
+
+  def statuses_to_delete(limit = 50, max_id = nil, min_id = nil)
+    scope = account.statuses
+    scope.merge!(old_enough_scope(max_id))
+    scope = scope.where(Status.arel_table[:id].gteq(min_id)) if min_id.present?
+    scope.merge!(without_popular_scope) unless min_favs.nil? && min_reblogs.nil?
+    scope.merge!(without_direct_scope) if keep_direct?
+    scope.merge!(without_pinned_scope) if keep_pinned?
+    scope.merge!(without_poll_scope) if keep_polls?
+    scope.merge!(without_media_scope) if keep_media?
+    scope.merge!(without_self_fav_scope) if keep_self_fav?
+    scope.merge!(without_self_bookmark_scope) if keep_self_bookmark?
+
+    scope.reorder(id: :asc).limit(limit)
+  end
+
+  # This computes a toot id such that:
+  # - the toot would be old enough to be candidate for deletion
+  # - there are at most EARLY_SEARCH_CUTOFF toots between the last inspected toot and this one
+  #
+  # The idea is to limit expensive SQL queries when an account has lots of toots excluded from
+  # deletion, while not starting anew on each run.
+  def compute_cutoff_id
+    min_id = last_inspected || 0
+    max_id = Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)
+    subquery = account.statuses.where(Status.arel_table[:id].gteq(min_id)).where(Status.arel_table[:id].lteq(max_id))
+    subquery = subquery.select(:id).reorder(id: :asc).limit(EARLY_SEARCH_CUTOFF)
+
+    # We're textually interpolating a subquery here as ActiveRecord seem to not provide
+    # a way to apply the limit to the subquery
+    Status.connection.execute("SELECT MAX(id) FROM (#{subquery.to_sql}) t").values.first.first
+  end
+
+  # The most important thing about `last_inspected` is that any toot older than it is guaranteed
+  # not to be kept by the policy regardless of its age.
+  def record_last_inspected(last_id)
+    redis.set("account_cleanup:#{account.id}", last_id, ex: 1.week.seconds)
+  end
+
+  def last_inspected
+    redis.get("account_cleanup:#{account.id}")&.to_i
+  end
+
+  def invalidate_last_inspected(status, action)
+    last_value = last_inspected
+    return if last_value.nil? || status.id > last_value || status.account_id != account_id
+
+    case action
+    when :unbookmark
+      return unless keep_self_bookmark?
+    when :unfav
+      return unless keep_self_fav?
+    when :unpin
+      return unless keep_pinned?
+    end
+
+    record_last_inspected(status.id)
+  end
+
+  private
+
+  def update_last_inspected
+    if EXCEPTION_BOOLS.map { |name| attribute_change_to_be_saved(name) }.compact.include?([true, false])
+      # Policy has been widened in such a way that any previously-inspected status
+      # may need to be deleted, so we'll have to start again.
+      redis.del("account_cleanup:#{account.id}")
+    end
+    if EXCEPTION_THRESHOLDS.map { |name| attribute_change_to_be_saved(name) }.compact.any? { |old, new| old.present? && (new.nil? || new > old) }
+      redis.del("account_cleanup:#{account.id}")
+    end
+  end
+
+  def validate_local_account
+    errors.add(:account, :invalid) unless account&.local?
+  end
+
+  def without_direct_scope
+    Status.where.not(visibility: :direct)
+  end
+
+  def old_enough_scope(max_id = nil)
+    # Filtering on `id` rather than `min_status_age` ago will treat
+    # non-snowflake statuses as older than they really are, but Mastodon
+    # has switched to snowflake IDs significantly over 2 years ago anyway.
+    max_id = [max_id, Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)].compact.min
+    Status.where(Status.arel_table[:id].lteq(max_id))
+  end
+
+  def without_self_fav_scope
+    Status.where('NOT EXISTS (SELECT * FROM favourites fav WHERE fav.account_id = statuses.account_id AND fav.status_id = statuses.id)')
+  end
+
+  def without_self_bookmark_scope
+    Status.where('NOT EXISTS (SELECT * FROM bookmarks bookmark WHERE bookmark.account_id = statuses.account_id AND bookmark.status_id = statuses.id)')
+  end
+
+  def without_pinned_scope
+    Status.where('NOT EXISTS (SELECT * FROM status_pins pin WHERE pin.account_id = statuses.account_id AND pin.status_id = statuses.id)')
+  end
+
+  def without_media_scope
+    Status.where('NOT EXISTS (SELECT * FROM media_attachments media WHERE media.status_id = statuses.id)')
+  end
+
+  def without_poll_scope
+    Status.where(poll_id: nil)
+  end
+
+  def without_popular_scope
+    scope = Status.left_joins(:status_stat)
+    scope = scope.where('COALESCE(status_stats.reblogs_count, 0) <= ?', min_reblogs) unless min_reblogs.nil?
+    scope = scope.where('COALESCE(status_stats.favourites_count, 0) <= ?', min_favs) unless min_favs.nil?
+    scope
+  end
+end
diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb
index 916261a17..f21ea714c 100644
--- a/app/models/bookmark.rb
+++ b/app/models/bookmark.rb
@@ -23,4 +23,12 @@ class Bookmark < ApplicationRecord
   before_validation do
     self.status = status.reblog if status&.reblog?
   end
+
+  after_destroy :invalidate_cleanup_info
+
+  def invalidate_cleanup_info
+    return unless status&.account_id == account_id && account.local?
+
+    account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unbookmark)
+  end
 end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index aaf371ebd..f2a4eae77 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -66,5 +66,8 @@ module AccountAssociations
 
     # Follow recommendations
     has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy
+
+    # Account statuses cleanup policy
+    has_one :statuses_cleanup_policy, class_name: 'AccountStatusesCleanupPolicy', inverse_of: :account, dependent: :destroy
   end
 end
diff --git a/app/models/favourite.rb b/app/models/favourite.rb
index 35028b7dd..ca8bce146 100644
--- a/app/models/favourite.rb
+++ b/app/models/favourite.rb
@@ -28,6 +28,7 @@ class Favourite < ApplicationRecord
 
   after_create :increment_cache_counters
   after_destroy :decrement_cache_counters
+  after_destroy :invalidate_cleanup_info
 
   private
 
@@ -39,4 +40,10 @@ class Favourite < ApplicationRecord
     return if association(:status).loaded? && status.marked_for_destruction?
     status&.decrement_count!(:favourites_count)
   end
+
+  def invalidate_cleanup_info
+    return unless status&.account_id == account_id && account.local?
+
+    account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unfav)
+  end
 end
diff --git a/app/models/status_pin.rb b/app/models/status_pin.rb
index afc76bded..93a0ea1c0 100644
--- a/app/models/status_pin.rb
+++ b/app/models/status_pin.rb
@@ -15,4 +15,12 @@ class StatusPin < ApplicationRecord
   belongs_to :status
 
   validates_with StatusPinValidator
+
+  after_destroy :invalidate_cleanup_info
+
+  def invalidate_cleanup_info
+    return unless status&.account_id == account_id && account.local?
+
+    account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unpin)
+  end
 end
diff --git a/app/services/account_statuses_cleanup_service.rb b/app/services/account_statuses_cleanup_service.rb
new file mode 100644
index 000000000..cbadecc63
--- /dev/null
+++ b/app/services/account_statuses_cleanup_service.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class AccountStatusesCleanupService < BaseService
+  # @param [AccountStatusesCleanupPolicy] account_policy
+  # @param [Integer] budget
+  # @return [Integer]
+  def call(account_policy, budget = 50)
+    return 0 unless account_policy.enabled?
+
+    cutoff_id = account_policy.compute_cutoff_id
+    return 0 if cutoff_id.blank?
+
+    num_deleted = 0
+    last_deleted = nil
+
+    account_policy.statuses_to_delete(budget, cutoff_id, account_policy.last_inspected).reorder(nil).find_each(order: :asc) do |status|
+      status.discard
+      RemovalWorker.perform_async(status.id, redraft: false)
+      num_deleted += 1
+      last_deleted = status.id
+    end
+
+    account_policy.record_last_inspected(last_deleted.presence || cutoff_id)
+
+    num_deleted
+  end
+end
diff --git a/app/views/statuses_cleanup/show.html.haml b/app/views/statuses_cleanup/show.html.haml
new file mode 100644
index 000000000..59de4b5aa
--- /dev/null
+++ b/app/views/statuses_cleanup/show.html.haml
@@ -0,0 +1,45 @@
+- content_for :page_title do
+  = t('settings.statuses_cleanup')
+
+- content_for :heading_actions do
+  = button_tag t('generic.save_changes'), class: 'button', form: 'edit_policy'
+
+= simple_form_for @policy, url: statuses_cleanup_path, method: :put, html: { id: 'edit_policy' } do |f|
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :enabled, as: :boolean, wrapper: :with_label, label: t('statuses_cleanup.enabled'), hint: t('statuses_cleanup.enabled_hint')
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :min_status_age, wrapper: :with_label, label: t('statuses_cleanup.min_age_label'), collection: AccountStatusesCleanupPolicy::ALLOWED_MIN_STATUS_AGE.map(&:to_i), label_method: lambda { |i| t("statuses_cleanup.min_age.#{i}") }, include_blank: false, hint: false
+
+  .flash-message= t('statuses_cleanup.explanation')
+
+  %h4= t('statuses_cleanup.exceptions')
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :keep_pinned, wrapper: :with_label, label: t('statuses_cleanup.keep_pinned'), hint: t('statuses_cleanup.keep_pinned_hint')
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :keep_direct, wrapper: :with_label, label: t('statuses_cleanup.keep_direct'), hint: t('statuses_cleanup.keep_direct_hint')
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :keep_self_fav, wrapper: :with_label, label: t('statuses_cleanup.keep_self_fav'), hint: t('statuses_cleanup.keep_self_fav_hint')
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :keep_self_bookmark, wrapper: :with_label, label: t('statuses_cleanup.keep_self_bookmark'), hint: t('statuses_cleanup.keep_self_bookmark_hint')
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :keep_polls, wrapper: :with_label, label: t('statuses_cleanup.keep_polls'), hint: t('statuses_cleanup.keep_polls_hint')
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :keep_media, wrapper: :with_label, label: t('statuses_cleanup.keep_media'), hint: t('statuses_cleanup.keep_media_hint')
+
+  %h4= t('statuses_cleanup.interaction_exceptions')
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :min_favs, wrapper: :with_label, label: t('statuses_cleanup.min_favs'), hint: t('statuses_cleanup.min_favs_hint'), input_html: { min: 1, placeholder: t('statuses_cleanup.ignore_favs') }
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :min_reblogs, wrapper: :with_label, label: t('statuses_cleanup.min_reblogs'), hint: t('statuses_cleanup.min_reblogs_hint'), input_html: { min: 1, placeholder: t('statuses_cleanup.ignore_reblogs') }
+
+  .flash-message= t('statuses_cleanup.interaction_exceptions_explanation')
diff --git a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
new file mode 100644
index 000000000..f42d4bca6
--- /dev/null
+++ b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+class Scheduler::AccountsStatusesCleanupScheduler
+  include Sidekiq::Worker
+
+  # This limit is mostly to be nice to the fediverse at large and not
+  # generate too much traffic.
+  # This also helps limiting the running time of the scheduler itself.
+  MAX_BUDGET         = 50
+
+  # This is an attempt to spread the load across instances, as various
+  # accounts are likely to have various followers.
+  PER_ACCOUNT_BUDGET = 5
+
+  # This is an attempt to limit the workload generated by status removal
+  # jobs to something the particular instance can handle.
+  PER_THREAD_BUDGET  = 5
+
+  # Those avoid loading an instance that is already under load
+  MAX_DEFAULT_SIZE    = 2
+  MAX_DEFAULT_LATENCY = 5
+  MAX_PUSH_SIZE       = 5
+  MAX_PUSH_LATENCY    = 10
+  # 'pull' queue has lower priority jobs, and it's unlikely that pushing
+  # deletes would cause much issues with this queue if it didn't cause issues
+  # with default and push. Yet, do not enqueue deletes if the instance is
+  # lagging behind too much.
+  MAX_PULL_SIZE       = 500
+  MAX_PULL_LATENCY    = 300
+
+  # This is less of an issue in general, but deleting old statuses is likely
+  # to cause delivery errors, and thus increase the number of jobs to be retried.
+  # This doesn't directly translate to load, but connection errors and a high
+  # number of dead instances may lead to this spiraling out of control if
+  # unchecked.
+  MAX_RETRY_SIZE = 50_000
+
+  sidekiq_options retry: 0, lock: :until_executed
+
+  def perform
+    return if under_load?
+
+    budget = compute_budget
+    first_policy_id = last_processed_id
+
+    loop do
+      num_processed_accounts = 0
+
+      scope = AccountStatusesCleanupPolicy.where(enabled: true)
+      scope.where(Account.arel_table[:id].gt(first_policy_id)) if first_policy_id.present?
+      scope.find_each(order: :asc) do |policy|
+        num_deleted = AccountStatusesCleanupService.new.call(policy, [budget, PER_ACCOUNT_BUDGET].min)
+        num_processed_accounts += 1 unless num_deleted.zero?
+        budget -= num_deleted
+        if budget.zero?
+          save_last_processed_id(policy.id)
+          break
+        end
+      end
+
+      # The idea here is to loop through all policies at least once until the budget is exhausted
+      # and start back after the last processed account otherwise
+      break if budget.zero? || (num_processed_accounts.zero? && first_policy_id.nil?)
+      first_policy_id = nil
+    end
+  end
+
+  def compute_budget
+    threads = Sidekiq::ProcessSet.new.filter { |x| x['queues'].include?('push') }.map { |x| x['concurrency'] }.sum
+    [PER_THREAD_BUDGET * threads, MAX_BUDGET].min
+  end
+
+  def under_load?
+    return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE
+    queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY)
+  end
+
+  private
+
+  def queue_under_load?(name, max_size, max_latency)
+    queue = Sidekiq::Queue.new(name)
+    queue.size > max_size || queue.latency > max_latency
+  end
+
+  def last_processed_id
+    Redis.current.get('account_statuses_cleanup_scheduler:last_account_id')
+  end
+
+  def save_last_processed_id(id)
+    if id.nil?
+      Redis.current.del('account_statuses_cleanup_scheduler:last_account_id')
+    else
+      Redis.current.set('account_statuses_cleanup_scheduler:last_account_id', id, ex: 1.hour.seconds)
+    end
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index af7266d86..be6052948 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1254,6 +1254,7 @@ en:
     preferences: Preferences
     profile: Profile
     relationships: Follows and followers
+    statuses_cleanup: Automated post deletion
     two_factor_authentication: Two-factor Auth
     webauthn_authentication: Security keys
   statuses:
@@ -1305,6 +1306,40 @@ en:
       public_long: Everyone can see
       unlisted: Unlisted
       unlisted_long: Everyone can see, but not listed on public timelines
+  statuses_cleanup:
+    enabled: Automatically delete old posts
+    enabled_hint: Automatically deletes your posts once they reach a specified age threshold, unless they match one of the exceptions below
+    exceptions: Exceptions
+    explanation: Because deleting posts is an expensive operation, this is done slowly over time when the server is not otherwise busy. For this reason, your posts may be deleted a while after they reach the age threshold.
+    ignore_favs: Ignore favourites
+    ignore_reblogs: Ignore boosts
+    interaction_exceptions: Exceptions based on interactions
+    interaction_exceptions_explanation: Note that there is no guarantee for posts to be deleted if they go below the favourite or boost threshold after having once gone over them.
+    keep_direct: Keep direct messages
+    keep_direct_hint: Doesn't delete any of your direct messages
+    keep_media: Keep posts with media attachments
+    keep_media_hint: Doesn't delete any of your posts that have media attachments
+    keep_pinned: Keep pinned posts
+    keep_pinned_hint: Doesn't delete any of your pinned posts
+    keep_polls: Keep polls
+    keep_polls_hint: Doesn't delete any of your polls
+    keep_self_bookmark: Keep posts you bookmarked
+    keep_self_bookmark_hint: Doesn't delete your own posts if you have bookmarked them
+    keep_self_fav: Keep posts you favourited
+    keep_self_fav_hint: Doesn't delete your own posts if you have favourited them
+    min_age:
+      '1209600': 2 weeks
+      '15778476': 6 months
+      '2629746': 1 month
+      '31556952': 1 year
+      '5259492': 2 months
+      '63113904': 2 years
+      '7889238': 3 months
+    min_age_label: Age threshold
+    min_favs: Keep posts favourited more than
+    min_favs_hint: Doesn't delete any of your posts that has received more than this amount of favourites. Leave blank to delete posts regardless of their number of favourites
+    min_reblogs: Keep posts boosted more than
+    min_reblogs_hint: Doesn't delete any of your posts that has been boosted more than this number of times. Leave blank to delete posts regardless of their number of boosts
   stream_entries:
     pinned: Pinned post
     reblogged: boosted
diff --git a/config/navigation.rb b/config/navigation.rb
index 5d1f55d74..37bfd7549 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -18,6 +18,7 @@ SimpleNavigation::Configuration.run do |navigation|
 
     n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_url, if: -> { current_user.functional? }
     n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
+    n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_url, if: -> { current_user.functional? }
 
     n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s|
       s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities}
diff --git a/config/routes.rb b/config/routes.rb
index 0c4b29546..8abc438fe 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -176,6 +176,7 @@ Rails.application.routes.draw do
   resources :invites, only: [:index, :create, :destroy]
   resources :filters, except: [:show]
   resource :relationships, only: [:show, :update]
+  resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update]
 
   get '/public', to: 'public_timelines#show', as: :public_timeline
   get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index a8e4c7feb..eab74338e 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -57,3 +57,7 @@
     cron: '0 * * * *'
     class: Scheduler::InstanceRefreshScheduler
     queue: scheduler
+  accounts_statuses_cleanup_scheduler:
+    interval: 1 minute
+    class: Scheduler::AccountsStatusesCleanupScheduler
+    queue: scheduler
diff --git a/db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb b/db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb
new file mode 100644
index 000000000..28cfb6ef5
--- /dev/null
+++ b/db/migrate/20210722120340_create_account_statuses_cleanup_policies.rb
@@ -0,0 +1,20 @@
+class CreateAccountStatusesCleanupPolicies < ActiveRecord::Migration[6.1]
+  def change
+    create_table :account_statuses_cleanup_policies do |t|
+      t.belongs_to :account, null: false, foreign_key: { on_delete: :cascade }
+      t.boolean :enabled, null: false, default: true
+      t.integer :min_status_age, null: false, default: 2.weeks.seconds
+      t.boolean :keep_direct, null: false, default: true
+      t.boolean :keep_pinned, null: false, default: true
+      t.boolean :keep_polls, null: false, default: false
+      t.boolean :keep_media, null: false, default: false
+      t.boolean :keep_self_fav, null: false, default: true
+      t.boolean :keep_self_bookmark, null: false, default: true
+      t.integer :min_favs, null: true
+      t.integer :min_reblogs, null: true
+
+      t.timestamps
+    end
+  end
+end
+
diff --git a/db/schema.rb b/db/schema.rb
index a0a98eb03..2376afff7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -114,6 +114,23 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
     t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
   end
 
+  create_table "account_statuses_cleanup_policies", force: :cascade do |t|
+    t.bigint "account_id", null: false
+    t.boolean "enabled", default: true, null: false
+    t.integer "min_status_age", default: 1209600, null: false
+    t.boolean "keep_direct", default: true, null: false
+    t.boolean "keep_pinned", default: true, null: false
+    t.boolean "keep_polls", default: false, null: false
+    t.boolean "keep_media", default: false, null: false
+    t.boolean "keep_self_fav", default: true, null: false
+    t.boolean "keep_self_bookmark", default: true, null: false
+    t.integer "min_favs"
+    t.integer "min_reblogs"
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["account_id"], name: "index_account_statuses_cleanup_policies_on_account_id"
+  end
+
   create_table "account_warning_presets", force: :cascade do |t|
     t.text "text", default: "", null: false
     t.datetime "created_at", null: false
@@ -984,6 +1001,7 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
   add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_pins", "accounts", on_delete: :cascade
   add_foreign_key "account_stats", "accounts", on_delete: :cascade
+  add_foreign_key "account_statuses_cleanup_policies", "accounts", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", on_delete: :nullify
   add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify
diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb
index 9e5bc7383..8e2d82a97 100644
--- a/lib/mastodon/snowflake.rb
+++ b/lib/mastodon/snowflake.rb
@@ -138,10 +138,11 @@ module Mastodon::Snowflake
       end
     end
 
-    def id_at(timestamp)
-      id  = timestamp.to_i * 1000 + rand(1000)
+    def id_at(timestamp, with_random: true)
+      id  = timestamp.to_i * 1000
+      id += rand(1000) if with_random
       id  = id << 16
-      id += rand(2**16)
+      id += rand(2**16) if with_random
       id
     end
 
diff --git a/spec/controllers/statuses_cleanup_controller_spec.rb b/spec/controllers/statuses_cleanup_controller_spec.rb
new file mode 100644
index 000000000..924709260
--- /dev/null
+++ b/spec/controllers/statuses_cleanup_controller_spec.rb
@@ -0,0 +1,27 @@
+require 'rails_helper'
+
+RSpec.describe StatusesCleanupController, type: :controller do
+  render_views
+
+  before do
+    @user = Fabricate(:user)
+    sign_in @user, scope: :user
+  end
+
+  describe "GET #show" do
+    it "returns http success" do
+      get :show
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'PUT #update' do
+    it 'updates the account status cleanup policy' do
+      put :update, params: { account_statuses_cleanup_policy: { enabled: true, min_status_age: 2.weeks.seconds, keep_direct: false, keep_polls: true } }
+      expect(response).to redirect_to(statuses_cleanup_path)
+      expect(@user.account.statuses_cleanup_policy.enabled).to eq true
+      expect(@user.account.statuses_cleanup_policy.keep_direct).to eq false
+      expect(@user.account.statuses_cleanup_policy.keep_polls).to eq true
+    end
+  end
+end
diff --git a/spec/fabricators/account_statuses_cleanup_policy_fabricator.rb b/spec/fabricators/account_statuses_cleanup_policy_fabricator.rb
new file mode 100644
index 000000000..29cf1d133
--- /dev/null
+++ b/spec/fabricators/account_statuses_cleanup_policy_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:account_statuses_cleanup_policy) do
+  account
+end
diff --git a/spec/models/account_statuses_cleanup_policy_spec.rb b/spec/models/account_statuses_cleanup_policy_spec.rb
new file mode 100644
index 000000000..63e9c5d20
--- /dev/null
+++ b/spec/models/account_statuses_cleanup_policy_spec.rb
@@ -0,0 +1,546 @@
+require 'rails_helper'
+
+RSpec.describe AccountStatusesCleanupPolicy, type: :model do
+  let(:account) { Fabricate(:account, username: 'alice', domain: nil) }
+
+  describe 'validation' do
+    it 'disallow remote accounts' do
+      account.update(domain: 'example.com')
+      account_statuses_cleanup_policy = Fabricate.build(:account_statuses_cleanup_policy, account: account)
+      account_statuses_cleanup_policy.valid?
+      expect(account_statuses_cleanup_policy).to model_have_error_on_field(:account)
+    end
+  end
+
+  describe 'save hooks' do
+    context 'when widening a policy' do
+      let!(:account_statuses_cleanup_policy) do
+        Fabricate(:account_statuses_cleanup_policy,
+          account: account,
+          keep_direct: true,
+          keep_pinned: true,
+          keep_polls: true,
+          keep_media: true,
+          keep_self_fav: true,
+          keep_self_bookmark: true,
+          min_favs: 1,
+          min_reblogs: 1
+        )
+      end
+
+      before do
+        account_statuses_cleanup_policy.record_last_inspected(42)
+      end
+
+      it 'invalidates last_inspected when widened because of keep_direct' do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of keep_pinned' do
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of keep_polls' do
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of keep_media' do
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of keep_self_fav' do
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of keep_self_bookmark' do
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of higher min_favs' do
+        account_statuses_cleanup_policy.min_favs = 5
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of disabled min_favs' do
+        account_statuses_cleanup_policy.min_favs = nil
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of higher min_reblogs' do
+        account_statuses_cleanup_policy.min_reblogs = 5
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+
+      it 'invalidates last_inspected when widened because of disable min_reblogs' do
+        account_statuses_cleanup_policy.min_reblogs = nil
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to be nil
+      end
+    end
+
+    context 'when narrowing a policy' do
+      let!(:account_statuses_cleanup_policy) do
+        Fabricate(:account_statuses_cleanup_policy,
+          account: account,
+          keep_direct: false,
+          keep_pinned: false,
+          keep_polls: false,
+          keep_media: false,
+          keep_self_fav: false,
+          keep_self_bookmark: false,
+          min_favs: nil,
+          min_reblogs: nil
+        )
+      end
+
+      it 'does not unnecessarily invalidate last_inspected' do
+        account_statuses_cleanup_policy.record_last_inspected(42)
+        account_statuses_cleanup_policy.keep_direct = true
+        account_statuses_cleanup_policy.keep_pinned = true
+        account_statuses_cleanup_policy.keep_polls = true
+        account_statuses_cleanup_policy.keep_media = true
+        account_statuses_cleanup_policy.keep_self_fav = true
+        account_statuses_cleanup_policy.keep_self_bookmark = true
+        account_statuses_cleanup_policy.min_favs = 5
+        account_statuses_cleanup_policy.min_reblogs = 5
+        account_statuses_cleanup_policy.save
+        expect(account_statuses_cleanup_policy.last_inspected).to eq 42
+      end
+    end
+  end
+
+  describe '#record_last_inspected' do
+    let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) }
+
+    it 'records the given id' do
+      account_statuses_cleanup_policy.record_last_inspected(42)
+      expect(account_statuses_cleanup_policy.last_inspected).to eq 42
+    end
+  end
+
+  describe '#invalidate_last_inspected' do
+    let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) }
+    let(:status) { Fabricate(:status, id: 10, account: account) }
+    subject { account_statuses_cleanup_policy.invalidate_last_inspected(status, action) }
+
+    before do
+      account_statuses_cleanup_policy.record_last_inspected(42)
+    end
+
+    context 'when the action is :unbookmark' do
+      let(:action) { :unbookmark }
+
+      context 'when the policy is not to keep self-bookmarked toots' do
+        before do
+          account_statuses_cleanup_policy.keep_self_bookmark = false
+        end
+
+        it 'does not change the recorded id' do
+          subject
+          expect(account_statuses_cleanup_policy.last_inspected).to eq 42
+        end
+      end
+
+      context 'when the policy is to keep self-bookmarked toots' do
+        before do
+          account_statuses_cleanup_policy.keep_self_bookmark = true
+        end
+
+        it 'records the older id' do
+          subject
+          expect(account_statuses_cleanup_policy.last_inspected).to eq 10
+        end
+      end
+    end
+
+    context 'when the action is :unfav' do
+      let(:action) { :unfav }
+
+      context 'when the policy is not to keep self-favourited toots' do
+        before do
+          account_statuses_cleanup_policy.keep_self_fav = false
+        end
+
+        it 'does not change the recorded id' do
+          subject
+          expect(account_statuses_cleanup_policy.last_inspected).to eq 42
+        end
+      end
+
+      context 'when the policy is to keep self-favourited toots' do
+        before do
+          account_statuses_cleanup_policy.keep_self_fav = true
+        end
+
+        it 'records the older id' do
+          subject
+          expect(account_statuses_cleanup_policy.last_inspected).to eq 10
+        end
+      end
+    end
+
+    context 'when the action is :unpin' do
+      let(:action) { :unpin }
+
+      context 'when the policy is not to keep pinned toots' do
+        before do
+          account_statuses_cleanup_policy.keep_pinned = false
+        end
+
+        it 'does not change the recorded id' do
+          subject
+          expect(account_statuses_cleanup_policy.last_inspected).to eq 42
+        end
+      end
+
+      context 'when the policy is to keep pinned toots' do
+        before do
+          account_statuses_cleanup_policy.keep_pinned = true
+        end
+
+        it 'records the older id' do
+          subject
+          expect(account_statuses_cleanup_policy.last_inspected).to eq 10
+        end
+      end
+    end
+
+    context 'when the status is more recent than the recorded inspected id' do
+      let(:action) { :unfav }
+      let(:status) { Fabricate(:status, account: account) }
+
+      it 'does not change the recorded id' do
+        subject
+        expect(account_statuses_cleanup_policy.last_inspected).to eq 42
+      end
+    end
+  end
+
+  describe '#compute_cutoff_id' do
+    let!(:unrelated_status)  { Fabricate(:status, created_at: 3.years.ago) }
+    let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) }
+
+    subject { account_statuses_cleanup_policy.compute_cutoff_id }
+
+    context 'when the account has posted multiple toots' do
+      let!(:very_old_status)   { Fabricate(:status, created_at: 3.years.ago, account: account) }
+      let!(:old_status)        { Fabricate(:status, created_at: 3.weeks.ago, account: account) }
+      let!(:recent_status)     { Fabricate(:status, created_at: 2.days.ago, account: account) }
+
+      it 'returns the most recent id that is still below policy age' do
+        expect(subject).to eq old_status.id
+      end
+    end
+
+    context 'when the account has not posted anything' do
+      it 'returns nil' do
+        expect(subject).to be_nil
+      end
+    end
+  end
+
+  describe '#statuses_to_delete' do
+    let!(:unrelated_status)  { Fabricate(:status, created_at: 3.years.ago) }
+    let!(:very_old_status)   { Fabricate(:status, created_at: 3.years.ago, account: account) }
+    let!(:pinned_status)     { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:direct_message)    { Fabricate(:status, created_at: 1.year.ago, account: account, visibility: :direct) }
+    let!(:self_faved)        { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:self_bookmarked)   { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:status_with_poll)  { Fabricate(:status, created_at: 1.year.ago, account: account, poll_attributes: { account: account, voters_count: 0, options: ['a', 'b'], expires_in: 2.days }) }
+    let!(:status_with_media) { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:faved4)            { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:faved5)            { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:reblogged4)        { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:reblogged5)        { Fabricate(:status, created_at: 1.year.ago, account: account) }
+    let!(:recent_status)     { Fabricate(:status, created_at: 2.days.ago, account: account) }
+
+    let!(:media_attachment)  { Fabricate(:media_attachment, account: account, status: status_with_media) }
+    let!(:status_pin)        { Fabricate(:status_pin, account: account, status: pinned_status) }
+    let!(:favourite)         { Fabricate(:favourite, account: account, status: self_faved) }
+    let!(:bookmark)          { Fabricate(:bookmark, account: account, status: self_bookmarked) }
+
+    let(:account_statuses_cleanup_policy) { Fabricate(:account_statuses_cleanup_policy, account: account) }
+
+    subject { account_statuses_cleanup_policy.statuses_to_delete }
+
+    before do
+      4.times { faved4.increment_count!(:favourites_count) }
+      5.times { faved5.increment_count!(:favourites_count) }
+      4.times { reblogged4.increment_count!(:reblogs_count) }
+      5.times { reblogged5.increment_count!(:reblogs_count) }
+    end
+
+    context 'when passed a max_id' do
+      let!(:old_status)               { Fabricate(:status, created_at: 1.year.ago, account: account) }
+      let!(:slightly_less_old_status) { Fabricate(:status, created_at: 6.months.ago, account: account) }
+
+      subject { account_statuses_cleanup_policy.statuses_to_delete(50, old_status.id).pluck(:id) }
+
+      it 'returns statuses including max_id' do
+        expect(subject).to include(old_status.id)
+      end
+
+      it 'returns statuses including older than max_id' do
+        expect(subject).to include(very_old_status.id)
+      end
+
+      it 'does not return statuses newer than max_id' do
+        expect(subject).to_not include(slightly_less_old_status.id)
+      end
+    end
+
+    context 'when passed a min_id' do
+      let!(:old_status)               { Fabricate(:status, created_at: 1.year.ago, account: account) }
+      let!(:slightly_less_old_status) { Fabricate(:status, created_at: 6.months.ago, account: account) }
+
+      subject { account_statuses_cleanup_policy.statuses_to_delete(50, recent_status.id, old_status.id).pluck(:id) }
+
+      it 'returns statuses including min_id' do
+        expect(subject).to include(old_status.id)
+      end
+
+      it 'returns statuses including newer than max_id' do
+        expect(subject).to include(slightly_less_old_status.id)
+      end
+
+      it 'does not return statuses older than min_id' do
+        expect(subject).to_not include(very_old_status.id)
+      end
+    end
+
+    context 'when passed a low limit' do
+      it 'only returns the limited number of items' do
+        expect(account_statuses_cleanup_policy.statuses_to_delete(1).count).to eq 1
+      end
+    end
+
+    context 'when policy is set to keep statuses more recent than 2 years' do
+      before do
+        account_statuses_cleanup_policy.min_status_age = 2.years.seconds
+      end
+
+      it 'does not return unrelated old status' do
+        expect(subject.pluck(:id)).to_not include(unrelated_status.id)
+      end
+
+      it 'returns only oldest status for deletion' do
+        expect(subject.pluck(:id)).to eq [very_old_status.id]
+      end
+    end
+
+    context 'when policy is set to keep DMs and reject everything else' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = true
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+      end
+
+      it 'does not return the old direct message for deletion' do
+        expect(subject.pluck(:id)).to_not include(direct_message.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is set to keep self-bookmarked toots and reject everything else' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.keep_self_bookmark = true
+      end
+
+      it 'does not return the old self-bookmarked message for deletion' do
+        expect(subject.pluck(:id)).to_not include(self_bookmarked.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is set to keep self-faved toots and reject everything else' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.keep_self_fav = true
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+      end
+
+      it 'does not return the old self-bookmarked message for deletion' do
+        expect(subject.pluck(:id)).to_not include(self_faved.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is set to keep toots with media and reject everything else' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.keep_media = true
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+      end
+
+      it 'does not return the old message with media for deletion' do
+        expect(subject.pluck(:id)).to_not include(status_with_media.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is set to keep toots with polls and reject everything else' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.keep_polls = true
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+      end
+
+      it 'does not return the old poll message for deletion' do
+        expect(subject.pluck(:id)).to_not include(status_with_poll.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is set to keep pinned toots and reject everything else' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.keep_pinned = true
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+      end
+
+      it 'does not return the old pinned message for deletion' do
+        expect(subject.pluck(:id)).to_not include(pinned_status.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is to not keep any special messages' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = false
+        account_statuses_cleanup_policy.keep_pinned = false
+        account_statuses_cleanup_policy.keep_polls = false
+        account_statuses_cleanup_policy.keep_media = false
+        account_statuses_cleanup_policy.keep_self_fav = false
+        account_statuses_cleanup_policy.keep_self_bookmark = false
+      end
+
+      it 'does not return the recent toot' do
+        expect(subject.pluck(:id)).to_not include(recent_status.id)
+      end
+
+      it 'does not return the unrelated toot' do
+        expect(subject.pluck(:id)).to_not include(unrelated_status.id)
+      end
+
+      it 'returns every other old status for deletion' do
+        expect(subject.pluck(:id)).to include(direct_message.id, very_old_status.id, pinned_status.id, self_faved.id, self_bookmarked.id, status_with_poll.id, status_with_media.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id)
+      end
+    end
+
+    context 'when policy is set to keep every category of toots' do
+      before do
+        account_statuses_cleanup_policy.keep_direct = true
+        account_statuses_cleanup_policy.keep_pinned = true
+        account_statuses_cleanup_policy.keep_polls = true
+        account_statuses_cleanup_policy.keep_media = true
+        account_statuses_cleanup_policy.keep_self_fav = true
+        account_statuses_cleanup_policy.keep_self_bookmark = true
+      end
+
+      it 'does not return unrelated old status' do
+        expect(subject.pluck(:id)).to_not include(unrelated_status.id)
+      end
+
+      it 'returns only normal statuses for deletion' do
+        expect(subject.pluck(:id).sort).to eq [very_old_status.id, faved4.id, faved5.id, reblogged4.id, reblogged5.id].sort
+      end
+    end
+
+    context 'when policy is to keep statuses with more than 4 boosts' do
+      before do
+        account_statuses_cleanup_policy.min_reblogs = 4
+      end
+
+      it 'does not return the recent toot' do
+        expect(subject.pluck(:id)).to_not include(recent_status.id)
+      end
+
+      it 'does not return the toot reblogged 5 times' do
+        expect(subject.pluck(:id)).to_not include(reblogged5.id)
+      end
+
+      it 'does not return the unrelated toot' do
+        expect(subject.pluck(:id)).to_not include(unrelated_status.id)
+      end
+
+      it 'returns old statuses not reblogged as much' do
+        expect(subject.pluck(:id)).to include(very_old_status.id, faved4.id, faved5.id, reblogged4.id)
+      end
+    end
+
+    context 'when policy is to keep statuses with more than 4 favs' do
+      before do
+        account_statuses_cleanup_policy.min_favs = 4
+      end
+
+      it 'does not return the recent toot' do
+        expect(subject.pluck(:id)).to_not include(recent_status.id)
+      end
+
+      it 'does not return the toot faved 5 times' do
+        expect(subject.pluck(:id)).to_not include(faved5.id)
+      end
+
+      it 'does not return the unrelated toot' do
+        expect(subject.pluck(:id)).to_not include(unrelated_status.id)
+      end
+
+      it 'returns old statuses not faved as much' do
+        expect(subject.pluck(:id)).to include(very_old_status.id, faved4.id, reblogged4.id, reblogged5.id)
+      end
+    end
+  end
+end
diff --git a/spec/services/account_statuses_cleanup_service_spec.rb b/spec/services/account_statuses_cleanup_service_spec.rb
new file mode 100644
index 000000000..257655c41
--- /dev/null
+++ b/spec/services/account_statuses_cleanup_service_spec.rb
@@ -0,0 +1,101 @@
+require 'rails_helper'
+
+describe AccountStatusesCleanupService, type: :service do
+  let(:account)           { Fabricate(:account, username: 'alice', domain: nil) }
+  let(:account_policy)    { Fabricate(:account_statuses_cleanup_policy, account: account) }
+  let!(:unrelated_status) { Fabricate(:status, created_at: 3.years.ago) }
+
+  describe '#call' do
+    context 'when the account has not posted anything' do
+      it 'returns 0 deleted toots' do
+        expect(subject.call(account_policy)).to eq 0
+      end
+    end
+
+    context 'when the account has posted several old statuses' do
+      let!(:very_old_status)    { Fabricate(:status, created_at: 3.years.ago, account: account) }
+      let!(:old_status)         { Fabricate(:status, created_at: 1.year.ago, account: account) }
+      let!(:another_old_status) { Fabricate(:status, created_at: 1.year.ago, account: account) }
+      let!(:recent_status)      { Fabricate(:status, created_at: 1.day.ago, account: account) }
+
+      context 'given a budget of 1' do
+        it 'reports 1 deleted toot' do
+          expect(subject.call(account_policy, 1)).to eq 1
+        end
+      end
+
+      context 'given a normal budget of 10' do
+        it 'reports 3 deleted statuses' do
+          expect(subject.call(account_policy, 10)).to eq 3
+        end
+
+        it 'records the last deleted id' do
+          subject.call(account_policy, 10)
+          expect(account_policy.last_inspected).to eq [old_status.id, another_old_status.id].max
+        end
+
+        it 'actually deletes the statuses' do
+          subject.call(account_policy, 10)
+          expect(Status.find_by(id: [very_old_status.id, old_status.id, another_old_status.id])).to be_nil
+        end
+      end
+
+      context 'when called repeatedly with a budget of 2' do
+        it 'reports 2 then 1 deleted statuses' do
+         expect(subject.call(account_policy, 2)).to eq 2
+         expect(subject.call(account_policy, 2)).to eq 1
+        end
+
+        it 'actually deletes the statuses in the expected order' do
+          subject.call(account_policy, 2)
+          expect(Status.find_by(id: very_old_status.id)).to be_nil
+          subject.call(account_policy, 2)
+          expect(Status.find_by(id: [very_old_status.id, old_status.id, another_old_status.id])).to be_nil
+        end
+      end
+
+      context 'when a self-faved toot is unfaved' do
+        let!(:self_faved) { Fabricate(:status, created_at: 6.months.ago, account: account) }
+        let!(:favourite)  { Fabricate(:favourite, account: account, status: self_faved) }
+
+        it 'deletes it once unfaved' do
+          expect(subject.call(account_policy, 20)).to eq 3
+          expect(Status.find_by(id: self_faved.id)).to_not be_nil
+          expect(subject.call(account_policy, 20)).to eq 0
+          favourite.destroy!
+          expect(subject.call(account_policy, 20)).to eq 1
+          expect(Status.find_by(id: self_faved.id)).to be_nil
+        end
+      end
+
+      context 'when there are more un-deletable old toots than the early search cutoff' do
+        before do
+          stub_const 'AccountStatusesCleanupPolicy::EARLY_SEARCH_CUTOFF', 5
+          # Old statuses that should be cut-off
+          10.times do
+            Fabricate(:status, created_at: 4.years.ago, visibility: :direct, account: account)
+          end
+          # New statuses that prevent cut-off id to reach the last status
+          10.times do
+            Fabricate(:status, created_at: 4.seconds.ago, visibility: :direct, account: account)
+          end
+        end
+
+        it 'reports 0 deleted statuses then 0 then 3 then 0 again' do
+          expect(subject.call(account_policy, 10)).to eq 0
+          expect(subject.call(account_policy, 10)).to eq 0
+          expect(subject.call(account_policy, 10)).to eq 3
+          expect(subject.call(account_policy, 10)).to eq 0
+        end
+
+        it 'never causes the recorded id to get higher than oldest deletable toot' do
+          subject.call(account_policy, 10)
+          subject.call(account_policy, 10)
+          subject.call(account_policy, 10)
+          subject.call(account_policy, 10)
+          expect(account_policy.last_inspected).to be < Mastodon::Snowflake.id_at(account_policy.min_status_age.seconds.ago, with_random: false)
+        end
+      end
+    end
+  end
+end
diff --git a/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb
new file mode 100644
index 000000000..8f20725c8
--- /dev/null
+++ b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb
@@ -0,0 +1,127 @@
+require 'rails_helper'
+
+describe Scheduler::AccountsStatusesCleanupScheduler do
+  subject { described_class.new }
+
+  let!(:account1)  { Fabricate(:account, domain: nil) }
+  let!(:account2)  { Fabricate(:account, domain: nil) }
+  let!(:account3)  { Fabricate(:account, domain: nil) }
+  let!(:account4)  { Fabricate(:account, domain: nil) }
+  let!(:remote)    { Fabricate(:account) }
+
+  let!(:policy1)   { Fabricate(:account_statuses_cleanup_policy, account: account1) }
+  let!(:policy2)   { Fabricate(:account_statuses_cleanup_policy, account: account3) }
+  let!(:policy3)   { Fabricate(:account_statuses_cleanup_policy, account: account4, enabled: false) }
+
+  let(:queue_size)       { 0 }
+  let(:queue_latency)    { 0 }
+  let(:process_set_stub) do
+    [
+      {
+        'concurrency' => 2,
+        'queues' => ['push', 'default'],
+      },
+    ]
+  end
+  let(:retry_size) { 0 }
+
+  before do
+    queue_stub = double
+    allow(queue_stub).to receive(:size).and_return(queue_size)
+    allow(queue_stub).to receive(:latency).and_return(queue_latency)
+    allow(Sidekiq::Queue).to receive(:new).and_return(queue_stub)
+    allow(Sidekiq::ProcessSet).to receive(:new).and_return(process_set_stub)
+
+    sidekiq_stats_stub = double
+    allow(sidekiq_stats_stub).to receive(:retry_size).and_return(retry_size)
+    allow(Sidekiq::Stats).to receive(:new).and_return(sidekiq_stats_stub)
+
+    # Create a bunch of old statuses
+    10.times do
+      Fabricate(:status, account: account1, created_at: 3.years.ago)
+      Fabricate(:status, account: account2, created_at: 3.years.ago)
+      Fabricate(:status, account: account3, created_at: 3.years.ago)
+      Fabricate(:status, account: account4, created_at: 3.years.ago)
+      Fabricate(:status, account: remote, created_at: 3.years.ago)
+    end
+
+    # Create a bunch of newer statuses
+    5.times do
+      Fabricate(:status, account: account1, created_at: 3.minutes.ago)
+      Fabricate(:status, account: account2, created_at: 3.minutes.ago)
+      Fabricate(:status, account: account3, created_at: 3.minutes.ago)
+      Fabricate(:status, account: account4, created_at: 3.minutes.ago)
+      Fabricate(:status, account: remote, created_at: 3.minutes.ago)
+    end
+  end
+
+  describe '#under_load?' do
+    context 'when nothing is queued' do
+      it 'returns false' do
+        expect(subject.under_load?).to be false
+      end
+    end
+
+    context 'when numerous jobs are queued' do
+      let(:queue_size)    { 5 }
+      let(:queue_latency) { 120 }
+
+      it 'returns true' do
+        expect(subject.under_load?).to be true
+      end
+    end
+
+    context 'when there is a huge amount of jobs to retry' do
+      let(:retry_size) { 1_000_000 }
+
+      it 'returns true' do
+        expect(subject.under_load?).to be true
+      end
+    end
+  end
+
+  describe '#get_budget' do
+    context 'on a single thread' do
+      let(:process_set_stub) { [ { 'concurrency' => 1, 'queues' => ['push', 'default'] } ] }
+
+      it 'returns a low value' do
+        expect(subject.compute_budget).to be < 10
+      end
+    end
+
+    context 'on a lot of threads' do
+      let(:process_set_stub) do
+        [
+          { 'concurrency' => 2, 'queues' => ['push', 'default'] },
+          { 'concurrency' => 2, 'queues' => ['push'] },
+          { 'concurrency' => 2, 'queues' => ['push'] },
+          { 'concurrency' => 2, 'queues' => ['push'] },
+        ]
+      end
+
+      it 'returns a larger value' do
+        expect(subject.compute_budget).to be > 10
+      end
+    end
+  end
+
+  describe '#perform' do
+    context 'when the budget is lower than the number of toots to delete' do
+      it 'deletes as many statuses as the given budget' do
+        expect { subject.perform }.to change { Status.count }.by(-subject.compute_budget)
+      end
+
+      it 'does not delete from accounts with no cleanup policy' do
+        expect { subject.perform }.to_not change { account2.statuses.count }
+      end
+
+      it 'does not delete from accounts with disabled cleanup policies' do
+        expect { subject.perform }.to_not change { account4.statuses.count }
+      end
+
+      it 'eventually deletes every deletable toot' do
+        expect { subject.perform; subject.perform; subject.perform; subject.perform }.to change { Status.count }.by(-20)
+      end
+    end
+  end
+end
-- 
cgit 


From 0cae6c07bb64b86bb1ac7188c11ddf182021aa5b Mon Sep 17 00:00:00 2001
From: Holger <holgerhuo@outlook.com>
Date: Fri, 20 Aug 2021 14:39:37 +0800
Subject: Fix #16603 (#16605)

Fix issue #16603 undefined method `serialize_payload' for Unsuspend Account Service error.
It seems that this service forgot to `include Payloadable` so that `serialize_payload` could not be found in this service.
---
 app/services/unsuspend_account_service.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'app/services')

diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb
index 949c670aa..b383f126a 100644
--- a/app/services/unsuspend_account_service.rb
+++ b/app/services/unsuspend_account_service.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class UnsuspendAccountService < BaseService
+  include Payloadable
   def call(account)
     @account = account
 
-- 
cgit 


From 9ac7e6fef770c0627c14d704fdf525c9515d6a6c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 20 Aug 2021 08:40:33 +0200
Subject: Fix remotely-suspended accounts' toots being merged back into
 timelines (#16628)

* Fix remotely-suspended accounts' toots being merged back into timelines

* Mark remotely-deleted accounts as remotely suspended
---
 app/services/resolve_account_service.rb   | 1 +
 app/services/unsuspend_account_service.rb | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 5400612bf..b266c019e 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -142,6 +142,7 @@ class ResolveAccountService < BaseService
   end
 
   def queue_deletion!
+    @account.suspend!(origin: :remote)
     AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true)
   end
 
diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb
index b383f126a..39d8a6ba7 100644
--- a/app/services/unsuspend_account_service.rb
+++ b/app/services/unsuspend_account_service.rb
@@ -8,7 +8,7 @@ class UnsuspendAccountService < BaseService
     unsuspend!
     refresh_remote_account!
 
-    return if @account.nil?
+    return if @account.nil? || @account.suspended?
 
     merge_into_home_timelines!
     merge_into_list_timelines!
-- 
cgit 


From a0d4129893c797f78d28ba9df5d35646f7bb0d80 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 26 Sep 2021 13:23:28 +0200
Subject: Refactor notifications to go through a separate stream in streaming
 API (#16765)

Eliminate need to have custom notifications filtering logic in the
streaming API code by publishing notifications into a separate stream
and then simply using the multi-stream capability to subscribe to that
stream when necessary
---
 app/services/notify_service.rb |  2 +-
 streaming/index.js             | 75 +++++++++++++++++++++++++-----------------
 2 files changed, 46 insertions(+), 31 deletions(-)

(limited to 'app/services')

diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index fc187db40..a1b5ca1e3 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -127,7 +127,7 @@ class NotifyService < BaseService
   def push_notification!
     return if @notification.activity.nil?
 
-    Redis.current.publish("timeline:#{@recipient.id}", Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, :notification)))
+    Redis.current.publish("timeline:#{@recipient.id}:notifications", Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, :notification)))
     send_push_notifications!
   end
 
diff --git a/streaming/index.js b/streaming/index.js
index 7bb645a13..67cd48b43 100644
--- a/streaming/index.js
+++ b/streaming/index.js
@@ -282,6 +282,14 @@ const startWorker = (workerId) => {
     next();
   };
 
+  /**
+   * @param {any} req
+   * @param {string[]} necessaryScopes
+   * @return {boolean}
+   */
+  const isInScope = (req, necessaryScopes) =>
+    req.scopes.some(scope => necessaryScopes.includes(scope));
+
   /**
    * @param {string} token
    * @param {any} req
@@ -314,7 +322,6 @@ const startWorker = (workerId) => {
         req.scopes = result.rows[0].scopes.split(' ');
         req.accountId = result.rows[0].account_id;
         req.chosenLanguages = result.rows[0].chosen_languages;
-        req.allowNotifications = req.scopes.some(scope => ['read', 'read:notifications'].includes(scope));
         req.deviceId = result.rows[0].device_id;
 
         resolve();
@@ -580,14 +587,12 @@ const startWorker = (workerId) => {
    * @param {function(string, string): void} output
    * @param {function(string[], function(string): void): void} attachCloseHandler
    * @param {boolean=} needsFiltering
-   * @param {boolean=} notificationOnly
    * @return {function(string): void}
    */
-  const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => {
+  const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false) => {
     const accountId  = req.accountId || req.remoteAddress;
-    const streamType = notificationOnly ? ' (notification)' : '';
 
-    log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}${streamType}`);
+    log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`);
 
     const listener = message => {
       const json = parseJSON(message);
@@ -605,14 +610,6 @@ const startWorker = (workerId) => {
         output(event, encodedPayload);
       };
 
-      if (notificationOnly && event !== 'notification') {
-        return;
-      }
-
-      if (event === 'notification' && !req.allowNotifications) {
-        return;
-      }
-
       // Only messages that may require filtering are statuses, since notifications
       // are already personalized and deletes do not matter
       if (!needsFiltering || event !== 'update') {
@@ -759,7 +756,7 @@ const startWorker = (workerId) => {
       const onSend = streamToHttp(req, res);
       const onEnd  = streamHttpEnd(req, subscriptionHeartbeat(channelIds));
 
-      streamFrom(channelIds, req, onSend, onEnd, options.needsFiltering, options.notificationOnly);
+      streamFrom(channelIds, req, onSend, onEnd, options.needsFiltering);
     }).catch(err => {
       log.verbose(req.requestId, 'Subscription error:', err.toString());
       httpNotFound(res);
@@ -775,74 +772,92 @@ const startWorker = (workerId) => {
    * @property {string} [only_media]
    */
 
+  /**
+   * @param {any} req
+   * @return {string[]}
+   */
+  const channelsForUserStream = req => {
+    const arr = [`timeline:${req.accountId}`];
+
+    if (isInScope(req, ['crypto']) && req.deviceId) {
+      arr.push(`timeline:${req.accountId}:${req.deviceId}`);
+    }
+
+    if (isInScope(req, ['read', 'read:notifications'])) {
+      arr.push(`timeline:${req.accountId}:notifications`);
+    }
+
+    return arr;
+  };
+
   /**
    * @param {any} req
    * @param {string} name
    * @param {StreamParams} params
-   * @return {Promise.<{ channelIds: string[], options: { needsFiltering: boolean, notificationOnly: boolean } }>}
+   * @return {Promise.<{ channelIds: string[], options: { needsFiltering: boolean } }>}
    */
   const channelNameToIds = (req, name, params) => new Promise((resolve, reject) => {
     switch(name) {
     case 'user':
       resolve({
-        channelIds: req.deviceId ? [`timeline:${req.accountId}`, `timeline:${req.accountId}:${req.deviceId}`] : [`timeline:${req.accountId}`],
-        options: { needsFiltering: false, notificationOnly: false },
+        channelIds: channelsForUserStream(req),
+        options: { needsFiltering: false },
       });
 
       break;
     case 'user:notification':
       resolve({
-        channelIds: [`timeline:${req.accountId}`],
-        options: { needsFiltering: false, notificationOnly: true },
+        channelIds: [`timeline:${req.accountId}:notifications`],
+        options: { needsFiltering: false },
       });
 
       break;
     case 'public':
       resolve({
         channelIds: ['timeline:public'],
-        options: { needsFiltering: true, notificationOnly: false },
+        options: { needsFiltering: true },
       });
 
       break;
     case 'public:local':
       resolve({
         channelIds: ['timeline:public:local'],
-        options: { needsFiltering: true, notificationOnly: false },
+        options: { needsFiltering: true },
       });
 
       break;
     case 'public:remote':
       resolve({
         channelIds: ['timeline:public:remote'],
-        options: { needsFiltering: true, notificationOnly: false },
+        options: { needsFiltering: true },
       });
 
       break;
     case 'public:media':
       resolve({
         channelIds: ['timeline:public:media'],
-        options: { needsFiltering: true, notificationOnly: false },
+        options: { needsFiltering: true },
       });
 
       break;
     case 'public:local:media':
       resolve({
         channelIds: ['timeline:public:local:media'],
-        options: { needsFiltering: true, notificationOnly: false },
+        options: { needsFiltering: true },
       });
 
       break;
     case 'public:remote:media':
       resolve({
         channelIds: ['timeline:public:remote:media'],
-        options: { needsFiltering: true, notificationOnly: false },
+        options: { needsFiltering: true },
       });
 
       break;
     case 'direct':
       resolve({
         channelIds: [`timeline:direct:${req.accountId}`],
-        options: { needsFiltering: false, notificationOnly: false },
+        options: { needsFiltering: false },
       });
 
       break;
@@ -852,7 +867,7 @@ const startWorker = (workerId) => {
       } else {
         resolve({
           channelIds: [`timeline:hashtag:${params.tag.toLowerCase()}`],
-          options: { needsFiltering: true, notificationOnly: false },
+          options: { needsFiltering: true },
         });
       }
 
@@ -863,7 +878,7 @@ const startWorker = (workerId) => {
       } else {
         resolve({
           channelIds: [`timeline:hashtag:${params.tag.toLowerCase()}:local`],
-          options: { needsFiltering: true, notificationOnly: false },
+          options: { needsFiltering: true },
         });
       }
 
@@ -872,7 +887,7 @@ const startWorker = (workerId) => {
       authorizeListAccess(params.list, req).then(() => {
         resolve({
           channelIds: [`timeline:list:${params.list}`],
-          options: { needsFiltering: false, notificationOnly: false },
+          options: { needsFiltering: false },
         });
       }).catch(() => {
         reject('Not authorized to stream this list');
@@ -919,7 +934,7 @@ const startWorker = (workerId) => {
 
       const onSend        = streamToWs(request, socket, streamNameFromChannelName(channelName, params));
       const stopHeartbeat = subscriptionHeartbeat(channelIds);
-      const listener      = streamFrom(channelIds, request, onSend, undefined, options.needsFiltering, options.notificationOnly);
+      const listener      = streamFrom(channelIds, request, onSend, undefined, options.needsFiltering);
 
       subscriptions[channelIds.join(';')] = {
         listener,
-- 
cgit 


From 216570ad988108c69b7105740ab382c544c2b034 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 14 Oct 2021 19:59:21 +0200
Subject: Fix scheduled statuses decreasing statuses counts (#16791)

* Add tests

* Fix scheduled statuses decreasing statuses counts

Fixes #16774
---
 app/models/status.rb                      |  2 +-
 app/services/post_status_service.rb       |  3 ++
 spec/services/post_status_service_spec.rb | 50 +++++++++++++++++--------------
 3 files changed, 31 insertions(+), 24 deletions(-)

(limited to 'app/services')

diff --git a/app/models/status.rb b/app/models/status.rb
index 847921ac2..3acf759f9 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -426,7 +426,7 @@ class Status < ApplicationRecord
   end
 
   def decrement_counter_caches
-    return if direct_visibility?
+    return if direct_visibility? || new_record?
 
     account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 0a383d6a3..85aaec4d6 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -74,6 +74,9 @@ class PostStatusService < BaseService
     status_for_validation = @account.statuses.build(status_attributes)
 
     if status_for_validation.valid?
+      # Marking the status as destroyed is necessary to prevent the status from being
+      # persisted when the associated media attachments get updated when creating the
+      # scheduled status.
       status_for_validation.destroy
 
       # The following transaction block is needed to wrap the UPDATEs to
diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb
index 147a59fc3..d21270c79 100644
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@@ -25,29 +25,33 @@ RSpec.describe PostStatusService, type: :service do
     expect(status.thread).to eq in_reply_to_status
   end
 
-  it 'schedules a status' do
-    account = Fabricate(:account)
-    future  = Time.now.utc + 2.hours
-
-    status = subject.call(account, text: 'Hi future!', scheduled_at: future)
-
-    expect(status).to be_a ScheduledStatus
-    expect(status.scheduled_at).to eq future
-    expect(status.params['text']).to eq 'Hi future!'
-  end
-
-  it 'does not immediately create a status when scheduling a status' do
-    account = Fabricate(:account)
-    media = Fabricate(:media_attachment)
-    future  = Time.now.utc + 2.hours
-
-    status = subject.call(account, text: 'Hi future!', media_ids: [media.id], scheduled_at: future)
-
-    expect(status).to be_a ScheduledStatus
-    expect(status.scheduled_at).to eq future
-    expect(status.params['text']).to eq 'Hi future!'
-    expect(media.reload.status).to be_nil
-    expect(Status.where(text: 'Hi future!').exists?).to be_falsey
+  context 'when scheduling a status' do
+    let!(:account)         { Fabricate(:account) }
+    let!(:future)          { Time.now.utc + 2.hours }
+    let!(:previous_status) { Fabricate(:status, account: account) }
+
+    it 'schedules a status' do
+      status = subject.call(account, text: 'Hi future!', scheduled_at: future)
+      expect(status).to be_a ScheduledStatus
+      expect(status.scheduled_at).to eq future
+      expect(status.params['text']).to eq 'Hi future!'
+    end
+
+    it 'does not immediately create a status' do
+      media = Fabricate(:media_attachment, account: account)
+      status = subject.call(account, text: 'Hi future!', media_ids: [media.id], scheduled_at: future)
+
+      expect(status).to be_a ScheduledStatus
+      expect(status.scheduled_at).to eq future
+      expect(status.params['text']).to eq 'Hi future!'
+      expect(status.params['media_ids']).to eq [media.id]
+      expect(media.reload.status).to be_nil
+      expect(Status.where(text: 'Hi future!').exists?).to be_falsey
+    end
+
+    it 'does not change statuses count' do
+      expect { subject.call(account, text: 'Hi future!', scheduled_at: future, thread: previous_status) }.not_to change { [account.statuses_count, previous_status.replies_count] }
+    end
   end
 
   it 'creates response to the original status of boost' do
-- 
cgit 


From 17f4e457b3a909522a230fd1f1f8f737e3faad87 Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Mon, 18 Oct 2021 19:02:35 +0900
Subject: Add remove from followers api (#16864)

* Add followed_by? to account_interactions

* Add RemoveFromFollowersService

* Fix AccountBatch to use RemoveFromFollowersService

* Add remove from followers API
---
 app/controllers/api/v1/accounts_controller.rb      |  9 +++--
 app/models/concerns/account_interactions.rb        |  4 +++
 app/models/form/account_batch.rb                   | 12 +------
 app/services/remove_from_followers_service.rb      | 25 ++++++++++++++
 config/routes.rb                                   |  1 +
 .../controllers/api/v1/accounts_controller_spec.rb | 20 ++++++++++++
 spec/models/concerns/account_interactions_spec.rb  | 17 ++++++++++
 spec/services/remove_from_follwers_service_spec.rb | 38 ++++++++++++++++++++++
 8 files changed, 113 insertions(+), 13 deletions(-)
 create mode 100644 app/services/remove_from_followers_service.rb
 create mode 100644 spec/services/remove_from_follwers_service_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index 95869f554..cbccd64f3 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::AccountsController < Api::BaseController
-  before_action -> { authorize_if_got_token! :read, :'read:accounts' }, except: [:create, :follow, :unfollow, :block, :unblock, :mute, :unmute]
-  before_action -> { doorkeeper_authorize! :follow, :'write:follows' }, only: [:follow, :unfollow]
+  before_action -> { authorize_if_got_token! :read, :'read:accounts' }, except: [:create, :follow, :unfollow, :remove_from_followers, :block, :unblock, :mute, :unmute]
+  before_action -> { doorkeeper_authorize! :follow, :'write:follows' }, only: [:follow, :unfollow, :remove_from_followers]
   before_action -> { doorkeeper_authorize! :follow, :'write:mutes' }, only: [:mute, :unmute]
   before_action -> { doorkeeper_authorize! :follow, :'write:blocks' }, only: [:block, :unblock]
   before_action -> { doorkeeper_authorize! :write, :'write:accounts' }, only: [:create]
@@ -53,6 +53,11 @@ class Api::V1::AccountsController < Api::BaseController
     render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships
   end
 
+  def remove_from_followers
+    RemoveFromFollowersService.new.call(current_user.account, @account)
+    render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships
+  end
+
   def unblock
     UnblockService.new.call(current_user.account, @account)
     render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships
diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index 8f19176a7..ad1665dc4 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -195,6 +195,10 @@ module AccountInteractions
     !following_anyone?
   end
 
+  def followed_by?(other_account)
+    other_account.following?(self)
+  end
+
   def blocking?(other_account)
     block_relationships.where(target_account: other_account).exists?
   end
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index 698933c9f..f1e1c8a65 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -43,9 +43,7 @@ class Form::AccountBatch
   end
 
   def remove_from_followers!
-    current_account.passive_relationships.where(account_id: account_ids).find_each do |follow|
-      reject_follow!(follow)
-    end
+    RemoveFromFollowersService.new.call(current_account, account_ids)
   end
 
   def block_domains!
@@ -62,14 +60,6 @@ class Form::AccountBatch
     Account.where(id: account_ids)
   end
 
-  def reject_follow!(follow)
-    follow.destroy
-
-    return unless follow.account.activitypub?
-
-    ActivityPub::DeliveryWorker.perform_async(Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), current_account.id, follow.account.inbox_url)
-  end
-
   def approve!
     users = accounts.includes(:user).map(&:user)
 
diff --git a/app/services/remove_from_followers_service.rb b/app/services/remove_from_followers_service.rb
new file mode 100644
index 000000000..3dac5467f
--- /dev/null
+++ b/app/services/remove_from_followers_service.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class RemoveFromFollowersService < BaseService
+  include Payloadable
+
+  def call(source_account, target_accounts)
+    source_account.passive_relationships.where(account_id: target_accounts).find_each do |follow|
+      follow.destroy
+
+      if source_account.local? && !follow.account.local? && follow.account.activitypub?
+        create_notification(follow)
+      end
+    end
+  end
+
+  private
+
+  def create_notification(follow)
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url)
+  end
+
+  def build_json(follow)
+    Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
+  end
+end
diff --git a/config/routes.rb b/config/routes.rb
index 6b6a6562d..86f699516 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -459,6 +459,7 @@ Rails.application.routes.draw do
         member do
           post :follow
           post :unfollow
+          post :remove_from_followers
           post :block
           post :unblock
           post :mute
diff --git a/spec/controllers/api/v1/accounts_controller_spec.rb b/spec/controllers/api/v1/accounts_controller_spec.rb
index d9ee37ffa..9a5a7c72a 100644
--- a/spec/controllers/api/v1/accounts_controller_spec.rb
+++ b/spec/controllers/api/v1/accounts_controller_spec.rb
@@ -168,6 +168,26 @@ RSpec.describe Api::V1::AccountsController, type: :controller do
     it_behaves_like 'forbidden for wrong scope', 'read:accounts'
   end
 
+  describe 'POST #remove_from_followers' do
+    let(:scopes) { 'write:follows' }
+    let(:other_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
+
+    before do
+      other_account.follow!(user.account)
+      post :remove_from_followers, params: { id: other_account.id }
+    end
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'removes the followed relation between user and target user' do
+      expect(user.account.followed_by?(other_account)).to be false
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'read:accounts'
+  end
+
   describe 'POST #block' do
     let(:scopes) { 'write:blocks' }
     let(:other_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb
index ca243ebc5..0369aff10 100644
--- a/spec/models/concerns/account_interactions_spec.rb
+++ b/spec/models/concerns/account_interactions_spec.rb
@@ -360,6 +360,23 @@ describe AccountInteractions do
     end
   end
 
+  describe '#followed_by?' do
+    subject { account.followed_by?(target_account) }
+
+    context 'followed by target_account' do
+      it 'returns true' do
+        account.passive_relationships.create(account: target_account)
+        is_expected.to be true
+      end
+    end
+
+    context 'not followed by target_account' do
+      it 'returns false' do
+        is_expected.to be false
+      end
+    end
+  end
+
   describe '#blocking?' do
     subject { account.blocking?(target_account) }
 
diff --git a/spec/services/remove_from_follwers_service_spec.rb b/spec/services/remove_from_follwers_service_spec.rb
new file mode 100644
index 000000000..a83f6f49a
--- /dev/null
+++ b/spec/services/remove_from_follwers_service_spec.rb
@@ -0,0 +1,38 @@
+require 'rails_helper'
+
+RSpec.describe RemoveFromFollowersService, type: :service do
+  let(:bob) { Fabricate(:account, username: 'bob') }
+
+  subject { RemoveFromFollowersService.new }
+
+  describe 'local' do
+    let(:sender) { Fabricate(:account, username: 'alice') }
+ 
+    before do
+      Follow.create(account: sender, target_account: bob)
+      subject.call(bob, sender)
+    end
+
+    it 'does not create follow relation' do
+      expect(bob.followed_by?(sender)).to be false
+    end
+  end
+
+  describe 'remote ActivityPub' do
+    let(:sender) { Fabricate(:account, username: 'alice', domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') }
+
+    before do
+      Follow.create(account: sender, target_account: bob)
+      stub_request(:post, sender.inbox_url).to_return(status: 200)
+      subject.call(bob, sender)
+    end
+
+    it 'does not create follow relation' do
+      expect(bob.followed_by?(sender)).to be false
+    end
+
+    it 'sends a reject activity' do
+      expect(a_request(:post, sender.inbox_url)).to have_been_made.once
+    end
+  end
+end
-- 
cgit 


From 3f9b28ce26ee6ec3b1657da01c9a90ecbcb4427a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 21 Oct 2021 01:14:04 +0200
Subject: Add support for fetching Create and Announce activities by URI
 (#16383)

* Add support for fetching Create and Announce activities by URI

This should improve compatibility with ZAP and offer a way to fetch boosts,
which is currently not possible.

* Add tests
---
 .../activitypub/fetch_remote_status_service.rb     | 29 +++++++++------
 .../fetch_remote_status_service_spec.rb            | 41 ++++++++++++++++++++++
 2 files changed, 60 insertions(+), 10 deletions(-)

(limited to 'app/services')

diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index cf4f62899..4f789d50b 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -13,7 +13,20 @@ class ActivityPub::FetchRemoteStatusService < BaseService
       end
     end
 
-    return if !(supported_context? && expected_type?) || actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id)
+    return unless supported_context?
+
+    actor_id = nil
+    activity_json = nil
+
+    if expected_object_type?
+      actor_id = value_or_id(first_of_value(@json['attributedTo']))
+      activity_json = { 'type' => 'Create', 'actor' => actor_id, 'object' => @json }
+    elsif expected_activity_type?
+      actor_id = value_or_id(first_of_value(@json['actor']))
+      activity_json = @json
+    end
+
+    return if activity_json.nil? || !trustworthy_attribution?(@json['id'], actor_id)
 
     actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account)
     actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update?(actor)
@@ -25,14 +38,6 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
   private
 
-  def activity_json
-    { 'type' => 'Create', 'actor' => actor_id, 'object' => @json }
-  end
-
-  def actor_id
-    value_or_id(first_of_value(@json['attributedTo']))
-  end
-
   def trustworthy_attribution?(uri, attributed_to)
     return false if uri.nil? || attributed_to.nil?
     Addressable::URI.parse(uri).normalized_host.casecmp(Addressable::URI.parse(attributed_to).normalized_host).zero?
@@ -42,7 +47,11 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     super(@json)
   end
 
-  def expected_type?
+  def expected_activity_type?
+    equals_or_includes_any?(@json['type'], %w(Create Announce))
+  end
+
+  def expected_object_type?
     equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
   end
 
diff --git a/spec/services/activitypub/fetch_remote_status_service_spec.rb b/spec/services/activitypub/fetch_remote_status_service_spec.rb
index 1ecc46952..ceba5f210 100644
--- a/spec/services/activitypub/fetch_remote_status_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_status_service_spec.rb
@@ -145,5 +145,46 @@ RSpec.describe ActivityPub::FetchRemoteStatusService, type: :service do
         expect(sender.statuses.first).to be_nil
       end
     end
+
+    context 'with a valid Create activity' do
+      let(:object) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: "https://#{valid_domain}/@foo/1234/create",
+          type: 'Create',
+          actor: ActivityPub::TagManager.instance.uri_for(sender),
+          object: note,
+        }
+      end
+
+      it 'creates status' do
+        status = sender.statuses.first
+
+        expect(status).to_not be_nil
+        expect(status.uri).to eq note[:id]
+        expect(status.text).to eq note[:content]
+      end
+    end
+
+    context 'with a Create activity with a mismatching id' do
+      let(:object) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: "https://#{valid_domain}/@foo/1234/create",
+          type: 'Create',
+          actor: ActivityPub::TagManager.instance.uri_for(sender),
+          object: {
+            id: "https://real.address/@foo/1234",
+            type: 'Note',
+            content: 'Lorem ipsum',
+            attributedTo: ActivityPub::TagManager.instance.uri_for(sender),
+          },
+        }
+      end
+
+      it 'does not create status' do
+        expect(sender.statuses.first).to be_nil
+      end
+    end
   end
 end
-- 
cgit 


From ec059317fad4fc5f1ed45a72cbae48edafcb586d Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 21 Oct 2021 20:39:35 +0200
Subject: Fix some link previews being incorrectly generated from other prior
 links (#16885)

* Add tests

* Fix some link previews being incorrectly generated from different prior links

PR #12403 added a cache to avoid redundant queries when the OEmbed endpoint can
be guessed from the URL. This caching mechanism is not perfectly correct as
there is no guarantee that all pages from a given domain share the same
OEmbed provider endpoint.

This PR prevents the FetchOEmbedService from caching OEmbed endpoint that
cannot be generalized by replacing a fully-qualified URL from the endpoint's
parameters, greatly reducing the number of incorrect cached generalizations.
---
 app/services/fetch_oembed_service.rb       |  5 +++-
 spec/fixtures/requests/oembed_youtube.html |  7 +++++
 spec/services/fetch_oembed_service_spec.rb | 41 ++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 spec/fixtures/requests/oembed_youtube.html

(limited to 'app/services')

diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb
index 60be9b9dc..4cbaa04c6 100644
--- a/app/services/fetch_oembed_service.rb
+++ b/app/services/fetch_oembed_service.rb
@@ -2,6 +2,7 @@
 
 class FetchOEmbedService
   ENDPOINT_CACHE_EXPIRES_IN = 24.hours.freeze
+  URL_REGEX                 = /(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i.freeze
 
   attr_reader :url, :options, :format, :endpoint_url
 
@@ -65,10 +66,12 @@ class FetchOEmbedService
   end
 
   def cache_endpoint!
+    return unless URL_REGEX.match?(@endpoint_url)
+
     url_domain = Addressable::URI.parse(@url).normalized_host
 
     endpoint_hash = {
-      endpoint: @endpoint_url.gsub(/(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i, '={url}'),
+      endpoint: @endpoint_url.gsub(URL_REGEX, '={url}'),
       format: @format,
     }
 
diff --git a/spec/fixtures/requests/oembed_youtube.html b/spec/fixtures/requests/oembed_youtube.html
new file mode 100644
index 000000000..1508e4dd9
--- /dev/null
+++ b/spec/fixtures/requests/oembed_youtube.html
@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <link rel="alternate" type="application/json+oembed" href="https://www.youtube.com/oembed?format=json&amp;url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DIPSbNdBmWKE" title="What is Mastodon?">
+  </head>
+  <body></body>
+</html>
diff --git a/spec/services/fetch_oembed_service_spec.rb b/spec/services/fetch_oembed_service_spec.rb
index a4262b040..88f0113ed 100644
--- a/spec/services/fetch_oembed_service_spec.rb
+++ b/spec/services/fetch_oembed_service_spec.rb
@@ -13,6 +13,32 @@ describe FetchOEmbedService, type: :service do
 
   describe 'discover_provider' do
     context 'when status code is 200 and MIME type is text/html' do
+      context 'when OEmbed endpoint contains URL as parameter' do
+        before do
+          stub_request(:get, 'https://www.youtube.com/watch?v=IPSbNdBmWKE').to_return(
+            status: 200,
+            headers: { 'Content-Type': 'text/html' },
+            body: request_fixture('oembed_youtube.html'),
+          )
+          stub_request(:get, 'https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DIPSbNdBmWKE').to_return(
+            status: 200,
+            headers: { 'Content-Type': 'text/html' },
+            body: request_fixture('oembed_json_empty.html')
+          )
+        end
+
+        it 'returns new OEmbed::Provider for JSON provider' do
+          subject.call('https://www.youtube.com/watch?v=IPSbNdBmWKE')
+          expect(subject.endpoint_url).to eq 'https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DIPSbNdBmWKE'
+          expect(subject.format).to eq :json
+        end
+
+        it 'stores URL template' do
+          subject.call('https://www.youtube.com/watch?v=IPSbNdBmWKE')
+          expect(Rails.cache.read('oembed_endpoint:www.youtube.com')[:endpoint]).to eq 'https://www.youtube.com/oembed?format=json&url={url}'
+        end
+      end
+
       context 'Both of JSON and XML provider are discoverable' do
         before do
           stub_request(:get, 'https://host.test/oembed.html').to_return(
@@ -33,6 +59,11 @@ describe FetchOEmbedService, type: :service do
           expect(subject.endpoint_url).to eq 'https://host.test/provider.xml'
           expect(subject.format).to eq :xml
         end
+
+        it 'does not cache OEmbed endpoint' do
+          subject.call('https://host.test/oembed.html', format: :xml)
+          expect(Rails.cache.exist?('oembed_endpoint:host.test')).to eq false
+        end
       end
 
       context 'JSON provider is discoverable while XML provider is not' do
@@ -49,6 +80,11 @@ describe FetchOEmbedService, type: :service do
           expect(subject.endpoint_url).to eq 'https://host.test/provider.json'
           expect(subject.format).to eq :json
         end
+
+        it 'does not cache OEmbed endpoint' do
+          subject.call('https://host.test/oembed.html')
+          expect(Rails.cache.exist?('oembed_endpoint:host.test')).to eq false
+        end
       end
 
       context 'XML provider is discoverable while JSON provider is not' do
@@ -65,6 +101,11 @@ describe FetchOEmbedService, type: :service do
           expect(subject.endpoint_url).to eq 'https://host.test/provider.xml'
           expect(subject.format).to eq :xml
         end
+
+        it 'does not cache OEmbed endpoint' do
+          subject.call('https://host.test/oembed.html')
+          expect(Rails.cache.exist?('oembed_endpoint:host.test')).to eq false
+        end
       end
 
       context 'Invalid XML provider is discoverable while JSON provider is not' do
-- 
cgit 


From 39cdf61ab7be267a374c472c230b315971ead43c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 5 Nov 2021 23:23:05 +0100
Subject: Add support for structured data and more OpenGraph tags to link cards
 (#16938)

Save preview cards under their canonical URL

Increase max redirects to follow from 2 to 3
---
 app/lib/link_details_extractor.rb             | 200 ++++++++++++++++++++++++++
 app/lib/request.rb                            |   2 +-
 app/services/fetch_link_card_service.rb       |  79 ++++------
 spec/lib/link_details_extractor_spec.rb       |  29 ++++
 spec/services/fetch_link_card_service_spec.rb |   2 +-
 5 files changed, 260 insertions(+), 52 deletions(-)
 create mode 100644 app/lib/link_details_extractor.rb
 create mode 100644 spec/lib/link_details_extractor_spec.rb

(limited to 'app/services')

diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
new file mode 100644
index 000000000..9df8a1320
--- /dev/null
+++ b/app/lib/link_details_extractor.rb
@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+class LinkDetailsExtractor
+  include ActionView::Helpers::TagHelper
+
+  class StructuredData
+    def initialize(data)
+      @data = data
+    end
+
+    def headline
+      json['headline']
+    end
+
+    def description
+      json['description']
+    end
+
+    def image
+      obj = first_of_value(json['image'])
+
+      return obj['url'] if obj.is_a?(Hash)
+
+      obj
+    end
+
+    def date_published
+      json['datePublished']
+    end
+
+    def date_modified
+      json['dateModified']
+    end
+
+    def author_name
+      author['name']
+    end
+
+    def author_url
+      author['url']
+    end
+
+    def publisher_name
+      publisher['name']
+    end
+
+    private
+
+    def author
+      first_of_value(json['author']) || {}
+    end
+
+    def publisher
+      first_of_value(json['publisher']) || {}
+    end
+
+    def first_of_value(arr)
+      arr.is_a?(Array) ? arr.first : arr
+    end
+
+    def json
+      @json ||= Oj.load(@data)
+    end
+  end
+
+  def initialize(original_url, html, html_charset)
+    @original_url = Addressable::URI.parse(original_url)
+    @html         = html
+    @html_charset = html_charset
+  end
+
+  def to_preview_card_attributes
+    {
+      title: title || '',
+      description: description || '',
+      image_remote_url: image,
+      type: type,
+      width: width || 0,
+      height: height || 0,
+      html: html || '',
+      provider_name: provider_name || '',
+      provider_url: provider_url || '',
+      author_name: author_name || '',
+      author_url: author_url || '',
+      embed_url: embed_url || '',
+    }
+  end
+
+  def type
+    player_url.present? ? :video : :link
+  end
+
+  def html
+    player_url.present? ? content_tag(:iframe, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
+  end
+
+  def width
+    opengraph_tag('twitter:player:width')
+  end
+
+  def height
+    opengraph_tag('twitter:player:height')
+  end
+
+  def title
+    structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first
+  end
+
+  def description
+    structured_data&.description || opengraph_tag('og:description') || meta_tag('description')
+  end
+
+  def image
+    valid_url_or_nil(opengraph_tag('og:image'))
+  end
+
+  def canonical_url
+    valid_url_or_nil(opengraph_tag('og:url') || link_tag('canonical'), same_origin_only: true) || @original_url.to_s
+  end
+
+  def provider_name
+    structured_data&.publisher_name || opengraph_tag('og:site_name')
+  end
+
+  def provider_url
+    valid_url_or_nil(host_to_url(opengraph_tag('og:site')))
+  end
+
+  def author_name
+    structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username')
+  end
+
+  def author_url
+    structured_data&.author_url
+  end
+
+  def embed_url
+    valid_url_or_nil(opengraph_tag('twitter:player:stream'))
+  end
+
+  private
+
+  def player_url
+    valid_url_or_nil(opengraph_tag('twitter:player'))
+  end
+
+  def host_to_url(str)
+    return if str.blank?
+
+    str.start_with?(/https?:\/\//) ? str : "http://#{str}"
+  end
+
+  def valid_url_or_nil(str, same_origin_only: false)
+    return if str.blank?
+
+    url = @original_url + Addressable::URI.parse(str)
+
+    return if url.host.blank? || !%w(http https).include?(url.scheme) || (same_origin_only && url.host != @original_url.host)
+
+    url.to_s
+  rescue Addressable::URI::InvalidURIError
+    nil
+  end
+
+  def link_tag(name)
+    document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
+  end
+
+  def opengraph_tag(name)
+    document.xpath("//meta[@property=\"#{name}\" or @name=\"#{name}\"]").map { |meta| meta['content'] }.first
+  end
+
+  def meta_tag(name)
+    document.xpath("//meta[@name=\"#{name}\"]").map { |meta| meta['content'] }.first
+  end
+
+  def structured_data
+    @structured_data ||= begin
+      json_ld = document.xpath('//script[@type="application/ld+json"]').map(&:content).first
+      json_ld.present? ? StructuredData.new(json_ld) : nil
+    end
+  end
+
+  def document
+    @document ||= Nokogiri::HTML(@html, nil, encoding)
+  end
+
+  def encoding
+    @encoding ||= begin
+      guess = detector.detect(@html, @html_charset)
+      guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+    end
+  end
+
+  def detector
+    @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
+      detector.strip_tags = true
+    end
+  end
+end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 125dee3ea..4289da933 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -94,7 +94,7 @@ class Request
     end
 
     def http_client
-      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 2)
+      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
     end
   end
 
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 5732ce8ac..51956ce7e 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -13,12 +13,12 @@ class FetchLinkCardService < BaseService
   }iox
 
   def call(status)
-    @status = status
-    @url    = parse_urls
+    @status       = status
+    @original_url = parse_urls
 
-    return if @url.nil? || @status.preview_cards.any?
+    return if @original_url.nil? || @status.preview_cards.any?
 
-    @url = @url.to_s
+    @url = @original_url.to_s
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
@@ -31,7 +31,7 @@ class FetchLinkCardService < BaseService
 
     attach_card if @card&.persisted?
   rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
-    Rails.logger.debug "Error fetching link #{@url}: #{e}"
+    Rails.logger.debug "Error fetching link #{@original_url}: #{e}"
     nil
   end
 
@@ -47,6 +47,12 @@ class FetchLinkCardService < BaseService
     return @html if defined?(@html)
 
     Request.new(:get, @url).add_headers('Accept' => 'text/html', 'User-Agent' => Mastodon::Version.user_agent + ' Bot').perform do |res|
+      # We follow redirects, and ideally we want to save the preview card for
+      # the destination URL and not any link shortener in-between, so here
+      # we set the URL to the one of the last response in the redirect chain
+      @url  = res.request.uri.to_s.to_s
+      @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url
+
       if res.code == 200 && res.mime_type == 'text/html'
         @html_charset = res.charset
         @html = res.body_with_limit
@@ -63,12 +69,15 @@ class FetchLinkCardService < BaseService
   end
 
   def parse_urls
-    if @status.local?
-      urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[1]).normalize }
-    else
-      html  = Nokogiri::HTML(@status.text)
-      links = html.css('a')
-      urls  = links.filter_map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.filter_map(&:normalize)
+    urls = begin
+      if @status.local?
+        @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[1]).normalize }
+      else
+        document = Nokogiri::HTML(@status.text)
+        links    = document.css('a')
+
+        links.filter_map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.filter_map(&:normalize)
+      end
     end
 
     urls.reject { |uri| bad_url?(uri) }.first
@@ -79,18 +88,16 @@ class FetchLinkCardService < BaseService
     uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
   end
 
-  # rubocop:disable Naming/MethodParameterName
-  def mention_link?(a)
+  def mention_link?(anchor)
     @status.mentions.any? do |mention|
-      a['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
+      anchor['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
     end
   end
 
-  def skip_link?(a)
+  def skip_link?(anchor)
     # Avoid links for hashtags and mentions (microformats)
-    a['rel']&.include?('tag') || a['class']&.match?(/u-url|h-card/) || mention_link?(a)
+    anchor['rel']&.include?('tag') || anchor['class']&.match?(/u-url|h-card/) || mention_link?(anchor)
   end
-  # rubocop:enable Naming/MethodParameterName
 
   def attempt_oembed
     service         = FetchOEmbedService.new
@@ -139,42 +146,14 @@ class FetchLinkCardService < BaseService
   def attempt_opengraph
     return if html.nil?
 
-    detector = CharlockHolmes::EncodingDetector.new
-    detector.strip_tags = true
-
-    guess      = detector.detect(@html, @html_charset)
-    encoding   = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
-    page       = Nokogiri::HTML(@html, nil, encoding)
-    player_url = meta_property(page, 'twitter:player')
-
-    if player_url && !bad_url?(Addressable::URI.parse(player_url))
-      @card.type   = :video
-      @card.width  = meta_property(page, 'twitter:player:width') || 0
-      @card.height = meta_property(page, 'twitter:player:height') || 0
-      @card.html   = content_tag(:iframe, nil, src: player_url,
-                                               width: @card.width,
-                                               height: @card.height,
-                                               allowtransparency: 'true',
-                                               scrolling: 'no',
-                                               frameborder: '0')
-    else
-      @card.type = :link
-    end
-
-    @card.title            = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
-    @card.description      = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
-    @card.image_remote_url = (Addressable::URI.parse(@url) + meta_property(page, 'og:image')).to_s if meta_property(page, 'og:image')
-
-    return if @card.title.blank? && @card.html.blank?
-
-    @card.save_with_optional_image!
-  end
+    link_details_extractor = LinkDetailsExtractor.new(@url, @html, @html_charset)
 
-  def meta_property(page, property)
-    page.at_xpath("//meta[contains(concat(' ', normalize-space(@property), ' '), ' #{property} ')]")&.attribute('content')&.value || page.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
+    @card = PreviewCard.find_or_initialize_by(url: link_details_extractor.canonical_url) if link_details_extractor.canonical_url != @card.url
+    @card.assign_attributes(link_details_extractor.to_preview_card_attributes)
+    @card.save_with_optional_image! unless @card.title.blank? && @card.html.blank?
   end
 
   def lock_options
-    { redis: Redis.current, key: "fetch:#{@url}", autorelease: 15.minutes.seconds }
+    { redis: Redis.current, key: "fetch:#{@original_url}", autorelease: 15.minutes.seconds }
   end
 end
diff --git a/spec/lib/link_details_extractor_spec.rb b/spec/lib/link_details_extractor_spec.rb
new file mode 100644
index 000000000..850857b2d
--- /dev/null
+++ b/spec/lib/link_details_extractor_spec.rb
@@ -0,0 +1,29 @@
+require 'rails_helper'
+
+RSpec.describe LinkDetailsExtractor do
+  let(:original_url) { '' }
+  let(:html) { '' }
+  let(:html_charset) { nil }
+
+  subject { described_class.new(original_url, html, html_charset) }
+
+  describe '#canonical_url' do
+    let(:original_url) { 'https://foo.com/article?bar=baz123' }
+
+    context 'when canonical URL points to another host' do
+      let(:html) { '<!doctype html><link rel="canonical" href="https://bar.com/different-article" />' }
+
+      it 'ignores the canonical URLs' do
+        expect(subject.canonical_url).to eq original_url
+      end
+    end
+
+    context 'when canonical URL points to the same host' do
+      let(:html) { '<!doctype html><link rel="canonical" href="https://foo.com/article" />' }
+
+      it 'ignores the canonical URLs' do
+        expect(subject.canonical_url).to eq 'https://foo.com/article'
+      end
+    end
+  end
+end
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 736a6078d..4914c2753 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 RSpec.describe FetchLinkCardService, type: :service do
-  subject { FetchLinkCardService.new }
+  subject { described_class.new }
 
   before do
     stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt'))
-- 
cgit 


From 3419d3ec84c3aa4f450265642e0a85dcdd3c36d0 Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Fri, 19 Nov 2021 06:02:08 +0900
Subject: Bump chewy from 5.2.0 to 7.2.3 (supports Elasticsearch 7.x) (#16915)

* Bump chewy from 5.2.0 to 7.2.2

* fix style (codeclimate)

* fix style

* fix style

* Bump chewy from 7.2.2 to 7.2.3
---
 Gemfile                                       |  2 +-
 Gemfile.lock                                  | 38 ++++++----
 app/chewy/accounts_index.rb                   | 24 +++----
 app/chewy/statuses_index.rb                   | 48 ++++++-------
 app/chewy/tags_index.rb                       | 16 ++---
 app/models/account.rb                         |  2 +-
 app/models/account_stat.rb                    |  2 +-
 app/models/bookmark.rb                        |  2 +-
 app/models/favourite.rb                       |  2 +-
 app/models/status.rb                          |  2 +-
 app/models/tag.rb                             |  2 +-
 app/services/batched_remove_status_service.rb |  2 +-
 app/services/delete_account_service.rb        |  4 +-
 config/initializers/chewy.rb                  | 20 ------
 lib/mastodon/search_cli.rb                    | 99 +++++++++++++--------------
 15 files changed, 128 insertions(+), 137 deletions(-)

(limited to 'app/services')

diff --git a/Gemfile b/Gemfile
index a420f0577..57d78f637 100644
--- a/Gemfile
+++ b/Gemfile
@@ -30,7 +30,7 @@ gem 'bootsnap', '~> 1.9.1', require: false
 gem 'browser'
 gem 'charlock_holmes', '~> 0.7.7'
 gem 'iso-639'
-gem 'chewy', '~> 5.2'
+gem 'chewy', '~> 7.2'
 gem 'cld3', '~> 3.4.2'
 gem 'devise', '~> 4.8'
 gem 'devise-two-factor', '~> 4.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 31dade486..30fe32a22 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -147,9 +147,9 @@ GEM
       activesupport
     cbor (0.5.9.6)
     charlock_holmes (0.7.7)
-    chewy (5.2.0)
+    chewy (7.2.3)
       activesupport (>= 5.2)
-      elasticsearch (>= 2.0.0)
+      elasticsearch (>= 7.12.0, < 7.14.0)
       elasticsearch-dsl
     chunky_png (1.4.0)
     cld3 (3.4.2)
@@ -197,13 +197,13 @@ GEM
       railties (>= 3.2)
     e2mmap (0.1.0)
     ed25519 (1.2.4)
-    elasticsearch (7.10.1)
-      elasticsearch-api (= 7.10.1)
-      elasticsearch-transport (= 7.10.1)
-    elasticsearch-api (7.10.1)
+    elasticsearch (7.13.3)
+      elasticsearch-api (= 7.13.3)
+      elasticsearch-transport (= 7.13.3)
+    elasticsearch-api (7.13.3)
       multi_json
-    elasticsearch-dsl (0.1.9)
-    elasticsearch-transport (7.10.1)
+    elasticsearch-dsl (0.1.10)
+    elasticsearch-transport (7.13.3)
       faraday (~> 1)
       multi_json
     encryptor (3.0.0)
@@ -214,11 +214,25 @@ GEM
     fabrication (2.22.0)
     faker (2.19.0)
       i18n (>= 1.6, < 2)
-    faraday (1.3.0)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
       faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
     faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
     fast_blank (1.0.1)
     fastimage (2.2.5)
     ffi (1.15.4)
@@ -539,7 +553,7 @@ GEM
     ruby-saml (1.13.0)
       nokogiri (>= 1.10.5)
       rexml
-    ruby2_keywords (0.0.4)
+    ruby2_keywords (0.0.5)
     rufus-scheduler (3.7.0)
       fugit (~> 1.1, >= 1.1.6)
     safety_net_attestation (0.4.0)
@@ -686,7 +700,7 @@ DEPENDENCIES
   capistrano-yarn (~> 2.0)
   capybara (~> 3.36)
   charlock_holmes (~> 0.7.7)
-  chewy (~> 5.2)
+  chewy (~> 7.2)
   cld3 (~> 3.4.2)
   climate_control (~> 0.2)
   color_diff (~> 0.1)
diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb
index b814e009e..6f9ea76e9 100644
--- a/app/chewy/accounts_index.rb
+++ b/app/chewy/accounts_index.rb
@@ -23,21 +23,21 @@ class AccountsIndex < Chewy::Index
     },
   }
 
-  define_type ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? } do
-    root date_detection: false do
-      field :id, type: 'long'
+  index_scope ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? }
 
-      field :display_name, type: 'text', analyzer: 'content' do
-        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-      end
+  root date_detection: false do
+    field :id, type: 'long'
 
-      field :acct, type: 'text', analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') } do
-        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-      end
+    field :display_name, type: 'text', analyzer: 'content' do
+      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+    end
 
-      field :following_count, type: 'long', value: ->(account) { account.following.local.count }
-      field :followers_count, type: 'long', value: ->(account) { account.followers.local.count }
-      field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    field :acct, type: 'text', analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') } do
+      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
     end
+
+    field :following_count, type: 'long', value: ->(account) { account.following.local.count }
+    field :followers_count, type: 'long', value: ->(account) { account.followers.local.count }
+    field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
   end
 end
diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
index 47cb856ea..1903c2ea3 100644
--- a/app/chewy/statuses_index.rb
+++ b/app/chewy/statuses_index.rb
@@ -31,36 +31,36 @@ class StatusesIndex < Chewy::Index
     },
   }
 
-  define_type ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll) do
-    crutch :mentions do |collection|
-      data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
-      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-    end
+  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll)
 
-    crutch :favourites do |collection|
-      data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-    end
+  crutch :mentions do |collection|
+    data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
+    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+  end
 
-    crutch :reblogs do |collection|
-      data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
-      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-    end
+  crutch :favourites do |collection|
+    data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
+    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+  end
 
-    crutch :bookmarks do |collection|
-      data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-    end
+  crutch :reblogs do |collection|
+    data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
+    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+  end
 
-    root date_detection: false do
-      field :id, type: 'long'
-      field :account_id, type: 'long'
+  crutch :bookmarks do |collection|
+    data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
+    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+  end
 
-      field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do
-        field :stemmed, type: 'text', analyzer: 'content'
-      end
+  root date_detection: false do
+    field :id, type: 'long'
+    field :account_id, type: 'long'
 
-      field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }
+    field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do
+      field :stemmed, type: 'text', analyzer: 'content'
     end
+
+    field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }
   end
 end
diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
index 300fc128f..f811a8d67 100644
--- a/app/chewy/tags_index.rb
+++ b/app/chewy/tags_index.rb
@@ -23,15 +23,15 @@ class TagsIndex < Chewy::Index
     },
   }
 
-  define_type ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? } do
-    root date_detection: false do
-      field :name, type: 'text', analyzer: 'content' do
-        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-      end
+  index_scope ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? }
 
-      field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
-      field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
-      field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+  root date_detection: false do
+    field :name, type: 'text', analyzer: 'content' do
+      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
     end
+
+    field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
+    field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+    field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
   end
 end
diff --git a/app/models/account.rb b/app/models/account.rb
index 291d3e571..d289c5e53 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -143,7 +143,7 @@ class Account < ApplicationRecord
 
   delegate :chosen_languages, to: :user, prefix: false, allow_nil: true
 
-  update_index('accounts#account', :self)
+  update_index('accounts', :self)
 
   def local?
     domain.nil?
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index e702fa4a4..b49827267 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -19,5 +19,5 @@ class AccountStat < ApplicationRecord
 
   belongs_to :account, inverse_of: :account_stat
 
-  update_index('accounts#account', :account)
+  update_index('accounts', :account)
 end
diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb
index f21ea714c..6334ef0df 100644
--- a/app/models/bookmark.rb
+++ b/app/models/bookmark.rb
@@ -13,7 +13,7 @@
 class Bookmark < ApplicationRecord
   include Paginable
 
-  update_index('statuses#status', :status) if Chewy.enabled?
+  update_index('statuses', :status) if Chewy.enabled?
 
   belongs_to :account, inverse_of: :bookmarks
   belongs_to :status,  inverse_of: :bookmarks
diff --git a/app/models/favourite.rb b/app/models/favourite.rb
index ca8bce146..2f355739a 100644
--- a/app/models/favourite.rb
+++ b/app/models/favourite.rb
@@ -13,7 +13,7 @@
 class Favourite < ApplicationRecord
   include Paginable
 
-  update_index('statuses#status', :status)
+  update_index('statuses', :status)
 
   belongs_to :account, inverse_of: :favourites
   belongs_to :status,  inverse_of: :favourites
diff --git a/app/models/status.rb b/app/models/status.rb
index c7f761bc6..749a23718 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -42,7 +42,7 @@ class Status < ApplicationRecord
   # will be based on current time instead of `created_at`
   attr_accessor :override_timestamps
 
-  update_index('statuses#status', :proper)
+  update_index('statuses', :proper)
 
   enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility
 
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 735c30608..dcce28391 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -39,7 +39,7 @@ class Tag < ApplicationRecord
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
   scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
-  update_index('tags#tag', :self)
+  update_index('tags', :self)
 
   def to_param
     name
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index b54bcae35..5000062e4 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -31,7 +31,7 @@ class BatchedRemoveStatusService < BaseService
 
     # Since we skipped all callbacks, we also need to manually
     # deindex the statuses
-    Chewy.strategy.current.update(StatusesIndex::Status, statuses_and_reblogs) if Chewy.enabled?
+    Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) if Chewy.enabled?
 
     return if options[:skip_side_effects]
 
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index d8270498a..ac571d7e2 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -189,7 +189,7 @@ class DeleteAccountService < BaseService
     @account.favourites.in_batches do |favourites|
       ids = favourites.pluck(:status_id)
       StatusStat.where(status_id: ids).update_all('favourites_count = GREATEST(0, favourites_count - 1)')
-      Chewy.strategy.current.update(StatusesIndex::Status, ids) if Chewy.enabled?
+      Chewy.strategy.current.update(StatusesIndex, ids) if Chewy.enabled?
       Rails.cache.delete_multi(ids.map { |id| "statuses/#{id}" })
       favourites.delete_all
     end
@@ -197,7 +197,7 @@ class DeleteAccountService < BaseService
 
   def purge_bookmarks!
     @account.bookmarks.in_batches do |bookmarks|
-      Chewy.strategy.current.update(StatusesIndex::Status, bookmarks.pluck(:status_id)) if Chewy.enabled?
+      Chewy.strategy.current.update(StatusesIndex, bookmarks.pluck(:status_id)) if Chewy.enabled?
       bookmarks.delete_all
     end
   end
diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb
index 820182a21..fbbcbbcde 100644
--- a/config/initializers/chewy.rb
+++ b/config/initializers/chewy.rb
@@ -37,23 +37,3 @@ end
 # Mastodon is run with hidden services enabled, because
 # ElasticSearch is *not* supposed to be accessed through a proxy
 Faraday.ignore_env_proxy = true
-
-# Elasticsearch 7.x workaround
-Elasticsearch::Transport::Client.prepend Module.new {
-  def search(arguments = {})
-    arguments[:rest_total_hits_as_int] = true
-    super arguments
-  end
-}
-
-Elasticsearch::API::Indices::IndicesClient.prepend Module.new {
-  def create(arguments = {})
-    arguments[:include_type_name] = true
-    super arguments
-  end
-
-  def put_mapping(arguments = {})
-    arguments[:include_type_name] = true
-    super arguments
-  end
-}
diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb
index 0126dfcff..2d1ca1c05 100644
--- a/lib/mastodon/search_cli.rb
+++ b/lib/mastodon/search_cli.rb
@@ -64,11 +64,7 @@ module Mastodon
       progress.title = 'Estimating workload '
 
       # Estimate the amount of data that has to be imported first
-      indices.each do |index|
-        index.types.each do |type|
-          progress.total = (progress.total || 0) + type.adapter.default_scope.count
-        end
-      end
+      progress.total = indices.sum { |index| index.adapter.default_scope.count }
 
       # Now import all the actual data. Mind that unlike chewy:sync, we don't
       # fetch and compare all record IDs from the database and the index to
@@ -80,67 +76,68 @@ module Mastodon
         batch_size     = 1_000
         slice_size     = (batch_size / options[:concurrency]).ceil
 
-        index.types.each do |type|
-          type.adapter.default_scope.reorder(nil).find_in_batches(batch_size: batch_size) do |batch|
-            futures = []
+        index.adapter.default_scope.reorder(nil).find_in_batches(batch_size: batch_size) do |batch|
+          futures = []
 
-            batch.each_slice(slice_size) do |records|
-              futures << Concurrent::Future.execute(executor: pool) do
-                begin
-                  if !progress.total.nil? && progress.progress + records.size > progress.total
-                    # The number of items has changed between start and now,
-                    # since there is no good way to predict the final count from
-                    # here, just change the progress bar to an indeterminate one
+          batch.each_slice(slice_size) do |records|
+            futures << Concurrent::Future.execute(executor: pool) do
+              begin
+                if !progress.total.nil? && progress.progress + records.size > progress.total
+                  # The number of items has changed between start and now,
+                  # since there is no good way to predict the final count from
+                  # here, just change the progress bar to an indeterminate one
 
-                    progress.total = nil
-                  end
+                  progress.total = nil
+                end
 
-                  grouped_records = nil
-                  bulk_body       = nil
-                  index_count     = 0
-                  delete_count    = 0
+                grouped_records = nil
+                bulk_body       = nil
+                index_count     = 0
+                delete_count    = 0
 
-                  ActiveRecord::Base.connection_pool.with_connection do
-                    grouped_records = type.adapter.send(:grouped_objects, records)
-                    bulk_body       = Chewy::Type::Import::BulkBuilder.new(type, **grouped_records).bulk_body
+                ActiveRecord::Base.connection_pool.with_connection do
+                  grouped_records = records.to_a.group_by do |record|
+                    index.adapter.send(:delete_from_index?, record) ? :delete : :to_index
                   end
 
-                  index_count  = grouped_records[:index].size  if grouped_records.key?(:index)
-                  delete_count = grouped_records[:delete].size if grouped_records.key?(:delete)
-
-                  # The following is an optimization for statuses specifically, since
-                  # we want to de-index statuses that cannot be searched by anybody,
-                  # but can't use Chewy's delete_if logic because it doesn't use
-                  # crutches and our searchable_by logic depends on them
-                  if type == StatusesIndex::Status
-                    bulk_body.map! do |entry|
-                      if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
-                        index_count  -= 1
-                        delete_count += 1
-
-                        { delete: entry[:index].except(:data) }
-                      else
-                        entry
-                      end
+                  bulk_body = Chewy::Index::Import::BulkBuilder.new(index, **grouped_records).bulk_body
+                end
+
+                index_count  = grouped_records[:to_index].size  if grouped_records.key?(:to_index)
+                delete_count = grouped_records[:delete].size    if grouped_records.key?(:delete)
+
+                # The following is an optimization for statuses specifically, since
+                # we want to de-index statuses that cannot be searched by anybody,
+                # but can't use Chewy's delete_if logic because it doesn't use
+                # crutches and our searchable_by logic depends on them
+                if index == StatusesIndex
+                  bulk_body.map! do |entry|
+                    if entry[:to_index] && entry.dig(:to_index, :data, 'searchable_by').blank?
+                      index_count  -= 1
+                      delete_count += 1
+
+                      { delete: entry[:to_index].except(:data) }
+                    else
+                      entry
                     end
                   end
+                end
 
-                  Chewy::Type::Import::BulkRequest.new(type).perform(bulk_body)
+                Chewy::Index::Import::BulkRequest.new(index).perform(bulk_body)
 
-                  progress.progress += records.size
+                progress.progress += records.size
 
-                  added.increment(index_count)
-                  removed.increment(delete_count)
+                added.increment(index_count)
+                removed.increment(delete_count)
 
-                  sleep 1
-                rescue => e
-                  progress.log pastel.red("Error importing #{index}: #{e}")
-                end
+                sleep 1
+              rescue => e
+                progress.log pastel.red("Error importing #{index}: #{e}")
               end
             end
-
-            futures.map(&:value)
           end
+
+          futures.map(&:value)
         end
       end
 
-- 
cgit 


From 6e50134a42cb303e6e42f89f9ddb5aacf83e7a6d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 25 Nov 2021 13:07:38 +0100
Subject: Add trending links (#16917)

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
---
 app/chewy/tags_index.rb                            |   2 +-
 app/controllers/admin/dashboard_controller.rb      |   2 +-
 app/controllers/admin/tags_controller.rb           |  76 +-----------
 .../links/preview_card_providers_controller.rb     |  41 +++++++
 app/controllers/admin/trends/links_controller.rb   |  45 ++++++++
 app/controllers/admin/trends/tags_controller.rb    |  41 +++++++
 .../api/v1/admin/dimensions_controller.rb          |   3 +-
 .../api/v1/admin/measures_controller.rb            |   3 +-
 .../api/v1/admin/trends/tags_controller.rb         |  16 +++
 app/controllers/api/v1/admin/trends_controller.rb  |  16 ---
 app/controllers/api/v1/trends/links_controller.rb  |  21 ++++
 app/controllers/api/v1/trends/tags_controller.rb   |  21 ++++
 app/controllers/api/v1/trends_controller.rb        |  15 ---
 app/helpers/admin/filter_helper.rb                 |   2 +
 app/helpers/languages_helper.rb                    |  94 +++++++++++++++
 app/helpers/settings_helper.rb                     |  89 +-------------
 .../mastodon/components/admin/Counter.js           |   5 +-
 .../mastodon/components/admin/Dimension.js         |   5 +-
 app/javascript/mastodon/components/admin/Trends.js |   2 +-
 app/javascript/styles/mastodon/accounts.scss       |  16 +++
 app/javascript/styles/mastodon/dashboard.scss      |  10 ++
 app/lib/activitypub/activity.rb                    |   2 -
 app/lib/activitypub/activity/announce.rb           |   5 +-
 app/lib/activitypub/activity/create.rb             |   7 +-
 app/lib/admin/metrics/dimension.rb                 |   9 +-
 app/lib/admin/metrics/dimension/base_dimension.rb  |  13 ++-
 .../admin/metrics/dimension/languages_dimension.rb |   4 +-
 .../metrics/dimension/tag_languages_dimension.rb   |  36 ++++++
 .../metrics/dimension/tag_servers_dimension.rb     |  35 ++++++
 app/lib/admin/metrics/measure.rb                   |  10 +-
 .../admin/metrics/measure/active_users_measure.rb  |   4 +-
 app/lib/admin/metrics/measure/base_measure.rb      |  15 ++-
 .../admin/metrics/measure/interactions_measure.rb  |   4 +-
 .../admin/metrics/measure/tag_accounts_measure.rb  |  41 +++++++
 .../admin/metrics/measure/tag_servers_measure.rb   |  47 ++++++++
 app/lib/admin/metrics/measure/tag_uses_measure.rb  |  41 +++++++
 app/lib/link_details_extractor.rb                  |  49 +++++++-
 app/mailers/admin_mailer.rb                        |  22 +++-
 app/models/account_statuses_cleanup_policy.rb      |   4 +-
 app/models/form/preview_card_batch.rb              |  65 +++++++++++
 app/models/form/preview_card_provider_batch.rb     |  33 ++++++
 app/models/form/tag_batch.rb                       |   8 +-
 app/models/preview_card.rb                         |  42 ++++++-
 app/models/preview_card_filter.rb                  |  53 +++++++++
 app/models/preview_card_provider.rb                |  57 +++++++++
 app/models/preview_card_provider_filter.rb         |  49 ++++++++
 app/models/tag.rb                                  |  23 +---
 app/models/tag_filter.rb                           |  56 +++++----
 app/models/trending_tags.rb                        | 128 ---------------------
 app/models/trends.rb                               |  27 +++++
 app/models/trends/base.rb                          |  80 +++++++++++++
 app/models/trends/history.rb                       |  98 ++++++++++++++++
 app/models/trends/links.rb                         | 117 +++++++++++++++++++
 app/models/trends/tags.rb                          | 111 ++++++++++++++++++
 app/policies/preview_card_policy.rb                |  11 ++
 app/policies/preview_card_provider_policy.rb       |  11 ++
 app/serializers/rest/trends/link_serializer.rb     |   5 +
 app/services/fetch_link_card_service.rb            |   3 +-
 app/services/post_status_service.rb                |   3 +-
 app/services/process_hashtags_service.rb           |   2 +-
 app/services/reblog_service.rb                     |  13 +--
 app/views/admin/dashboard/index.html.haml          |   2 +-
 app/views/admin/tags/_tag.html.haml                |  19 ---
 app/views/admin/tags/index.html.haml               |  74 ------------
 app/views/admin/tags/show.html.haml                |  68 +++++++----
 .../admin/trends/links/_preview_card.html.haml     |  30 +++++
 app/views/admin/trends/links/index.html.haml       |  41 +++++++
 .../_preview_card_provider.html.haml               |  16 +++
 .../links/preview_card_providers/index.html.haml   |  43 +++++++
 app/views/admin/trends/tags/_tag.html.haml         |  24 ++++
 app/views/admin/trends/tags/index.html.haml        |  38 ++++++
 app/views/admin_mailer/new_trending_links.text.erb |  16 +++
 app/views/admin_mailer/new_trending_tag.text.erb   |   5 -
 app/views/admin_mailer/new_trending_tags.text.erb  |  16 +++
 app/views/application/_sidebar.html.haml           |   2 +-
 app/workers/scheduler/trending_tags_scheduler.rb   |  11 --
 app/workers/scheduler/trends/refresh_scheduler.rb  |  11 ++
 .../trends/review_notifications_scheduler.rb       |  11 ++
 config/brakeman.ignore                             | 112 +++++++++++++-----
 config/locales/en.yml                              |  73 +++++++++---
 config/locales/simple_form.en.yml                  |   4 +-
 config/navigation.rb                               |   6 +-
 config/routes.rb                                   |  36 ++++--
 config/sidekiq.yml                                 |   8 +-
 ...20211031031021_create_preview_card_providers.rb |  12 ++
 ...20211112011713_add_language_to_preview_cards.rb |   7 ++
 ...0211115032527_add_trendable_to_preview_cards.rb |   5 +
 ...0211123212714_add_link_type_to_preview_cards.rb |   5 +
 db/schema.rb                                       |  21 +++-
 lib/mastodon/snowflake.rb                          |   5 +-
 lib/tasks/repo.rake                                |   2 +-
 spec/controllers/admin/tags_controller_spec.rb     |  12 --
 .../api/v1/trends/tags_controller_spec.rb          |  22 ++++
 spec/controllers/api/v1/trends_controller_spec.rb  |  18 ---
 spec/helpers/languages_helper_spec.rb              |  17 +++
 spec/helpers/settings_helper_spec.rb               |  22 ----
 spec/mailers/previews/admin_mailer_preview.rb      |  10 ++
 spec/models/trending_tags_spec.rb                  |  68 -----------
 spec/models/trends/tags_spec.rb                    |  67 +++++++++++
 99 files changed, 2088 insertions(+), 739 deletions(-)
 create mode 100644 app/controllers/admin/trends/links/preview_card_providers_controller.rb
 create mode 100644 app/controllers/admin/trends/links_controller.rb
 create mode 100644 app/controllers/admin/trends/tags_controller.rb
 create mode 100644 app/controllers/api/v1/admin/trends/tags_controller.rb
 delete mode 100644 app/controllers/api/v1/admin/trends_controller.rb
 create mode 100644 app/controllers/api/v1/trends/links_controller.rb
 create mode 100644 app/controllers/api/v1/trends/tags_controller.rb
 delete mode 100644 app/controllers/api/v1/trends_controller.rb
 create mode 100644 app/helpers/languages_helper.rb
 create mode 100644 app/lib/admin/metrics/dimension/tag_languages_dimension.rb
 create mode 100644 app/lib/admin/metrics/dimension/tag_servers_dimension.rb
 create mode 100644 app/lib/admin/metrics/measure/tag_accounts_measure.rb
 create mode 100644 app/lib/admin/metrics/measure/tag_servers_measure.rb
 create mode 100644 app/lib/admin/metrics/measure/tag_uses_measure.rb
 create mode 100644 app/models/form/preview_card_batch.rb
 create mode 100644 app/models/form/preview_card_provider_batch.rb
 create mode 100644 app/models/preview_card_filter.rb
 create mode 100644 app/models/preview_card_provider.rb
 create mode 100644 app/models/preview_card_provider_filter.rb
 delete mode 100644 app/models/trending_tags.rb
 create mode 100644 app/models/trends.rb
 create mode 100644 app/models/trends/base.rb
 create mode 100644 app/models/trends/history.rb
 create mode 100644 app/models/trends/links.rb
 create mode 100644 app/models/trends/tags.rb
 create mode 100644 app/policies/preview_card_policy.rb
 create mode 100644 app/policies/preview_card_provider_policy.rb
 create mode 100644 app/serializers/rest/trends/link_serializer.rb
 delete mode 100644 app/views/admin/tags/_tag.html.haml
 delete mode 100644 app/views/admin/tags/index.html.haml
 create mode 100644 app/views/admin/trends/links/_preview_card.html.haml
 create mode 100644 app/views/admin/trends/links/index.html.haml
 create mode 100644 app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml
 create mode 100644 app/views/admin/trends/links/preview_card_providers/index.html.haml
 create mode 100644 app/views/admin/trends/tags/_tag.html.haml
 create mode 100644 app/views/admin/trends/tags/index.html.haml
 create mode 100644 app/views/admin_mailer/new_trending_links.text.erb
 delete mode 100644 app/views/admin_mailer/new_trending_tag.text.erb
 create mode 100644 app/views/admin_mailer/new_trending_tags.text.erb
 delete mode 100644 app/workers/scheduler/trending_tags_scheduler.rb
 create mode 100644 app/workers/scheduler/trends/refresh_scheduler.rb
 create mode 100644 app/workers/scheduler/trends/review_notifications_scheduler.rb
 create mode 100644 db/migrate/20211031031021_create_preview_card_providers.rb
 create mode 100644 db/migrate/20211112011713_add_language_to_preview_cards.rb
 create mode 100644 db/migrate/20211115032527_add_trendable_to_preview_cards.rb
 create mode 100644 db/migrate/20211123212714_add_link_type_to_preview_cards.rb
 create mode 100644 spec/controllers/api/v1/trends/tags_controller_spec.rb
 delete mode 100644 spec/controllers/api/v1/trends_controller_spec.rb
 create mode 100644 spec/helpers/languages_helper_spec.rb
 delete mode 100644 spec/helpers/settings_helper_spec.rb
 delete mode 100644 spec/models/trending_tags_spec.rb
 create mode 100644 spec/models/trends/tags_spec.rb

(limited to 'app/services')

diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
index f811a8d67..f9db2b03a 100644
--- a/app/chewy/tags_index.rb
+++ b/app/chewy/tags_index.rb
@@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index
     end
 
     field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
-    field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+    field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } }
     field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
   end
 end
diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb
index cbfff2707..f0a935411 100644
--- a/app/controllers/admin/dashboard_controller.rb
+++ b/app/controllers/admin/dashboard_controller.rb
@@ -4,7 +4,7 @@ module Admin
   class DashboardController < BaseController
     def index
       @system_checks         = Admin::SystemCheck.perform
-      @time_period           = (1.month.ago.to_date...Time.now.utc.to_date)
+      @time_period           = (29.days.ago.to_date...Time.now.utc.to_date)
       @pending_users_count   = User.pending.count
       @pending_reports_count = Report.unresolved.count
       @pending_tags_count    = Tag.pending_review.count
diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index eed4feea2..749e2f144 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -2,38 +2,12 @@
 
 module Admin
   class TagsController < BaseController
-    before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
-    before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
-    before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
-
-    def index
-      authorize :tag, :index?
-
-      @tags = filtered_tags.page(params[:page])
-      @form = Form::TagBatch.new
-    end
-
-    def batch
-      @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
-      @form.save
-    rescue ActionController::ParameterMissing
-      flash[:alert] = I18n.t('admin.accounts.no_account_selected')
-    ensure
-      redirect_to admin_tags_path(filter_params)
-    end
-
-    def approve_all
-      Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
-      redirect_to admin_tags_path(filter_params)
-    end
-
-    def reject_all
-      Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
-      redirect_to admin_tags_path(filter_params)
-    end
+    before_action :set_tag
 
     def show
       authorize @tag, :show?
+
+      @time_period = (6.days.ago.to_date...Time.now.utc.to_date)
     end
 
     def update
@@ -52,52 +26,8 @@ module Admin
       @tag = Tag.find(params[:id])
     end
 
-    def set_usage_by_domain
-      @usage_by_domain = @tag.statuses
-                             .with_public_visibility
-                             .excluding_silenced_accounts
-                             .where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day)))
-                             .joins(:account)
-                             .group('accounts.domain')
-                             .reorder(statuses_count: :desc)
-                             .pluck(Arel.sql('accounts.domain, count(*) AS statuses_count'))
-    end
-
-    def set_counters
-      @accounts_today = @tag.history.first[:accounts]
-      @accounts_week  = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" })
-    end
-
-    def filtered_tags
-      TagFilter.new(filter_params).results
-    end
-
-    def filter_params
-      params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
-    end
-
     def tag_params
       params.require(:tag).permit(:name, :trendable, :usable, :listable)
     end
-
-    def current_week_days
-      now = Time.now.utc.beginning_of_day.to_date
-
-      (Date.commercial(now.cwyear, now.cweek)..now).map do |date|
-        date.to_time(:utc).beginning_of_day.to_i
-      end
-    end
-
-    def form_tag_batch_params
-      params.require(:form_tag_batch).permit(:action, tag_ids: [])
-    end
-
-    def action_from_button
-      if params[:approve]
-        'approve'
-      elsif params[:reject]
-        'reject'
-      end
-    end
   end
 end
diff --git a/app/controllers/admin/trends/links/preview_card_providers_controller.rb b/app/controllers/admin/trends/links/preview_card_providers_controller.rb
new file mode 100644
index 000000000..2c26e03f3
--- /dev/null
+++ b/app/controllers/admin/trends/links/preview_card_providers_controller.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController
+  def index
+    authorize :preview_card_provider, :index?
+
+    @preview_card_providers = filtered_preview_card_providers.page(params[:page])
+    @form = Form::PreviewCardProviderBatch.new
+  end
+
+  def batch
+    @form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
+    @form.save
+  rescue ActionController::ParameterMissing
+    flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+  ensure
+    redirect_to admin_trends_links_preview_card_providers_path(filter_params)
+  end
+
+  private
+
+  def filtered_preview_card_providers
+    PreviewCardProviderFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
+  end
+
+  def form_preview_card_provider_batch_params
+    params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
+  end
+
+  def action_from_button
+    if params[:approve]
+      'approve'
+    elsif params[:reject]
+      'reject'
+    end
+  end
+end
diff --git a/app/controllers/admin/trends/links_controller.rb b/app/controllers/admin/trends/links_controller.rb
new file mode 100644
index 000000000..619b37deb
--- /dev/null
+++ b/app/controllers/admin/trends/links_controller.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class Admin::Trends::LinksController < Admin::BaseController
+  def index
+    authorize :preview_card, :index?
+
+    @preview_cards = filtered_preview_cards.page(params[:page])
+    @form          = Form::PreviewCardBatch.new
+  end
+
+  def batch
+    @form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
+    @form.save
+  rescue ActionController::ParameterMissing
+    flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+  ensure
+    redirect_to admin_trends_links_path(filter_params)
+  end
+
+  private
+
+  def filtered_preview_cards
+    PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
+  end
+
+  def filter_params
+    params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
+  end
+
+  def form_preview_card_batch_params
+    params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
+  end
+
+  def action_from_button
+    if params[:approve]
+      'approve'
+    elsif params[:approve_all]
+      'approve_all'
+    elsif params[:reject]
+      'reject'
+    elsif params[:reject_all]
+      'reject_all'
+    end
+  end
+end
diff --git a/app/controllers/admin/trends/tags_controller.rb b/app/controllers/admin/trends/tags_controller.rb
new file mode 100644
index 000000000..91ff33d40
--- /dev/null
+++ b/app/controllers/admin/trends/tags_controller.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Trends::TagsController < Admin::BaseController
+  def index
+    authorize :tag, :index?
+
+    @tags = filtered_tags.page(params[:page])
+    @form = Form::TagBatch.new
+  end
+
+  def batch
+    @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
+    @form.save
+  rescue ActionController::ParameterMissing
+    flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+  ensure
+    redirect_to admin_trends_tags_path(filter_params)
+  end
+
+  private
+
+  def filtered_tags
+    TagFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
+  end
+
+  def form_tag_batch_params
+    params.require(:form_tag_batch).permit(:action, tag_ids: [])
+  end
+
+  def action_from_button
+    if params[:approve]
+      'approve'
+    elsif params[:reject]
+      'reject'
+    end
+  end
+end
diff --git a/app/controllers/api/v1/admin/dimensions_controller.rb b/app/controllers/api/v1/admin/dimensions_controller.rb
index 170596d27..5e8f0f89f 100644
--- a/app/controllers/api/v1/admin/dimensions_controller.rb
+++ b/app/controllers/api/v1/admin/dimensions_controller.rb
@@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController
       params[:keys],
       params[:start_at],
       params[:end_at],
-      params[:limit]
+      params[:limit],
+      params
     )
   end
 end
diff --git a/app/controllers/api/v1/admin/measures_controller.rb b/app/controllers/api/v1/admin/measures_controller.rb
index a3ac6fe85..f28191753 100644
--- a/app/controllers/api/v1/admin/measures_controller.rb
+++ b/app/controllers/api/v1/admin/measures_controller.rb
@@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController
     @measures = Admin::Metrics::Measure.retrieve(
       params[:keys],
       params[:start_at],
-      params[:end_at]
+      params[:end_at],
+      params
     )
   end
 end
diff --git a/app/controllers/api/v1/admin/trends/tags_controller.rb b/app/controllers/api/v1/admin/trends/tags_controller.rb
new file mode 100644
index 000000000..3653d1dd1
--- /dev/null
+++ b/app/controllers/api/v1/admin/trends/tags_controller.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::Trends::TagsController < Api::BaseController
+  before_action :require_staff!
+  before_action :set_tags
+
+  def index
+    render json: @tags, each_serializer: REST::Admin::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = Trends.tags.get(false, limit_param(10))
+  end
+end
diff --git a/app/controllers/api/v1/admin/trends_controller.rb b/app/controllers/api/v1/admin/trends_controller.rb
deleted file mode 100644
index e32ab5d2c..000000000
--- a/app/controllers/api/v1/admin/trends_controller.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::Admin::TrendsController < Api::BaseController
-  before_action :require_staff!
-  before_action :set_trends
-
-  def index
-    render json: @trends, each_serializer: REST::Admin::TagSerializer
-  end
-
-  private
-
-  def set_trends
-    @trends = TrendingTags.get(10, filtered: false)
-  end
-end
diff --git a/app/controllers/api/v1/trends/links_controller.rb b/app/controllers/api/v1/trends/links_controller.rb
new file mode 100644
index 000000000..1c3ab1e1c
--- /dev/null
+++ b/app/controllers/api/v1/trends/links_controller.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Api::V1::Trends::LinksController < Api::BaseController
+  before_action :set_links
+
+  def index
+    render json: @links, each_serializer: REST::Trends::LinkSerializer
+  end
+
+  private
+
+  def set_links
+    @links = begin
+      if Setting.trends
+        Trends.links.get(true, limit_param(10))
+      else
+        []
+      end
+    end
+  end
+end
diff --git a/app/controllers/api/v1/trends/tags_controller.rb b/app/controllers/api/v1/trends/tags_controller.rb
new file mode 100644
index 000000000..947b53de2
--- /dev/null
+++ b/app/controllers/api/v1/trends/tags_controller.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Api::V1::Trends::TagsController < Api::BaseController
+  before_action :set_tags
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = begin
+      if Setting.trends
+        Trends.tags.get(true, limit_param(10))
+      else
+        []
+      end
+    end
+  end
+end
diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb
deleted file mode 100644
index c875e9041..000000000
--- a/app/controllers/api/v1/trends_controller.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::TrendsController < Api::BaseController
-  before_action :set_tags
-
-  def index
-    render json: @tags, each_serializer: REST::TagSerializer
-  end
-
-  private
-
-  def set_tags
-    @tags = TrendingTags.get(limit_param(10))
-  end
-end
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index ba0ca9638..5f69f176a 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -6,6 +6,8 @@ module Admin::FilterHelper
     CustomEmojiFilter::KEYS,
     ReportFilter::KEYS,
     TagFilter::KEYS,
+    PreviewCardProviderFilter::KEYS,
+    PreviewCardFilter::KEYS,
     InstanceFilter::KEYS,
     InviteFilter::KEYS,
     RelationshipFilter::KEYS,
diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb
new file mode 100644
index 000000000..730724208
--- /dev/null
+++ b/app/helpers/languages_helper.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module LanguagesHelper
+  HUMAN_LOCALES = {
+    af: 'Afrikaans',
+    ar: 'العربية',
+    ast: 'Asturianu',
+    bg: 'Български',
+    bn: 'বাংলা',
+    br: 'Breton',
+    ca: 'Català',
+    co: 'Corsu',
+    cs: 'Čeština',
+    cy: 'Cymraeg',
+    da: 'Dansk',
+    de: 'Deutsch',
+    el: 'Ελληνικά',
+    en: 'English',
+    eo: 'Esperanto',
+    'es-AR': 'Español (Argentina)',
+    'es-MX': 'Español (México)',
+    es: 'Español',
+    et: 'Eesti',
+    eu: 'Euskara',
+    fa: 'فارسی',
+    fi: 'Suomi',
+    fr: 'Français',
+    ga: 'Gaeilge',
+    gd: 'Gàidhlig',
+    gl: 'Galego',
+    he: 'עברית',
+    hi: 'हिन्दी',
+    hr: 'Hrvatski',
+    hu: 'Magyar',
+    hy: 'Հայերեն',
+    id: 'Bahasa Indonesia',
+    io: 'Ido',
+    is: 'Íslenska',
+    it: 'Italiano',
+    ja: '日本語',
+    ka: 'ქართული',
+    kab: 'Taqbaylit',
+    kk: 'Қазақша',
+    kmr: 'Kurmancî',
+    kn: 'ಕನ್ನಡ',
+    ko: '한국어',
+    ku: 'سۆرانی',
+    lt: 'Lietuvių',
+    lv: 'Latviešu',
+    mk: 'Македонски',
+    ml: 'മലയാളം',
+    mr: 'मराठी',
+    ms: 'Bahasa Melayu',
+    nl: 'Nederlands',
+    nn: 'Nynorsk',
+    no: 'Norsk',
+    oc: 'Occitan',
+    pl: 'Polski',
+    'pt-BR': 'Português (Brasil)',
+    'pt-PT': 'Português (Portugal)',
+    pt: 'Português',
+    ro: 'Română',
+    ru: 'Русский',
+    sa: 'संस्कृतम्',
+    sc: 'Sardu',
+    si: 'සිංහල',
+    sk: 'Slovenčina',
+    sl: 'Slovenščina',
+    sq: 'Shqip',
+    'sr-Latn': 'Srpski (latinica)',
+    sr: 'Српски',
+    sv: 'Svenska',
+    ta: 'தமிழ்',
+    te: 'తెలుగు',
+    th: 'ไทย',
+    tr: 'Türkçe',
+    uk: 'Українська',
+    ur: 'اُردُو',
+    vi: 'Tiếng Việt',
+    zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
+    'zh-CN': '简体中文',
+    'zh-HK': '繁體中文（香港）',
+    'zh-TW': '繁體中文（臺灣）',
+    zh: '中文',
+  }.freeze
+
+  def human_locale(locale)
+    if locale == 'und'
+      I18n.t('generic.none')
+    else
+      HUMAN_LOCALES[locale.to_sym] || locale
+    end
+  end
+end
diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index ac4c18746..23739d1cd 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -1,95 +1,8 @@
 # frozen_string_literal: true
 
 module SettingsHelper
-  HUMAN_LOCALES = {
-    af: 'Afrikaans',
-    ar: 'العربية',
-    ast: 'Asturianu',
-    bg: 'Български',
-    bn: 'বাংলা',
-    br: 'Breton',
-    ca: 'Català',
-    co: 'Corsu',
-    cs: 'Čeština',
-    cy: 'Cymraeg',
-    da: 'Dansk',
-    de: 'Deutsch',
-    el: 'Ελληνικά',
-    en: 'English',
-    eo: 'Esperanto',
-    'es-AR': 'Español (Argentina)',
-    'es-MX': 'Español (México)',
-    es: 'Español',
-    et: 'Eesti',
-    eu: 'Euskara',
-    fa: 'فارسی',
-    fi: 'Suomi',
-    fr: 'Français',
-    ga: 'Gaeilge',
-    gd: 'Gàidhlig',
-    gl: 'Galego',
-    he: 'עברית',
-    hi: 'हिन्दी',
-    hr: 'Hrvatski',
-    hu: 'Magyar',
-    hy: 'Հայերեն',
-    id: 'Bahasa Indonesia',
-    io: 'Ido',
-    is: 'Íslenska',
-    it: 'Italiano',
-    ja: '日本語',
-    ka: 'ქართული',
-    kab: 'Taqbaylit',
-    kk: 'Қазақша',
-    kmr: 'Kurmancî',
-    kn: 'ಕನ್ನಡ',
-    ko: '한국어',
-    ku: 'سۆرانی',
-    lt: 'Lietuvių',
-    lv: 'Latviešu',
-    mk: 'Македонски',
-    ml: 'മലയാളം',
-    mr: 'मराठी',
-    ms: 'Bahasa Melayu',
-    nl: 'Nederlands',
-    nn: 'Nynorsk',
-    no: 'Norsk',
-    oc: 'Occitan',
-    pl: 'Polski',
-    'pt-BR': 'Português (Brasil)',
-    'pt-PT': 'Português (Portugal)',
-    pt: 'Português',
-    ro: 'Română',
-    ru: 'Русский',
-    sa: 'संस्कृतम्',
-    sc: 'Sardu',
-    si: 'සිංහල',
-    sk: 'Slovenčina',
-    sl: 'Slovenščina',
-    sq: 'Shqip',
-    'sr-Latn': 'Srpski (latinica)',
-    sr: 'Српски',
-    sv: 'Svenska',
-    ta: 'தமிழ்',
-    te: 'తెలుగు',
-    th: 'ไทย',
-    tr: 'Türkçe',
-    uk: 'Українська',
-    ur: 'اُردُو',
-    vi: 'Tiếng Việt',
-    zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
-    'zh-CN': '简体中文',
-    'zh-HK': '繁體中文（香港）',
-    'zh-TW': '繁體中文（臺灣）',
-    zh: '中文',
-  }.freeze
-
-  def human_locale(locale)
-    HUMAN_LOCALES[locale]
-  end
-
   def filterable_languages
-    LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?))
+    LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?))
   end
 
   def hash_to_object(hash)
diff --git a/app/javascript/mastodon/components/admin/Counter.js b/app/javascript/mastodon/components/admin/Counter.js
index cda572dce..047e864b2 100644
--- a/app/javascript/mastodon/components/admin/Counter.js
+++ b/app/javascript/mastodon/components/admin/Counter.js
@@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent {
     end_at: PropTypes.string.isRequired,
     label: PropTypes.string.isRequired,
     href: PropTypes.string,
+    params: PropTypes.object,
   };
 
   state = {
@@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent {
   };
 
   componentDidMount () {
-    const { measure, start_at, end_at } = this.props;
+    const { measure, start_at, end_at, params } = this.props;
 
-    api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => {
+    api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => {
       this.setState({
         loading: false,
         data: res.data,
diff --git a/app/javascript/mastodon/components/admin/Dimension.js b/app/javascript/mastodon/components/admin/Dimension.js
index ac6dbd1c7..977c8208d 100644
--- a/app/javascript/mastodon/components/admin/Dimension.js
+++ b/app/javascript/mastodon/components/admin/Dimension.js
@@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent {
     end_at: PropTypes.string.isRequired,
     limit: PropTypes.number.isRequired,
     label: PropTypes.string.isRequired,
+    params: PropTypes.object,
   };
 
   state = {
@@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent {
   };
 
   componentDidMount () {
-    const { start_at, end_at, dimension, limit } = this.props;
+    const { start_at, end_at, dimension, limit, params } = this.props;
 
-    api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => {
+    api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => {
       this.setState({
         loading: false,
         data: res.data,
diff --git a/app/javascript/mastodon/components/admin/Trends.js b/app/javascript/mastodon/components/admin/Trends.js
index 46307a28a..635bdf37d 100644
--- a/app/javascript/mastodon/components/admin/Trends.js
+++ b/app/javascript/mastodon/components/admin/Trends.js
@@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent {
   componentDidMount () {
     const { limit } = this.props;
 
-    api().get('/api/v1/admin/trends', { params: { limit } }).then(res => {
+    api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => {
       this.setState({
         loading: false,
         data: res.data,
diff --git a/app/javascript/styles/mastodon/accounts.scss b/app/javascript/styles/mastodon/accounts.scss
index 2c78e81be..b8a6c8018 100644
--- a/app/javascript/styles/mastodon/accounts.scss
+++ b/app/javascript/styles/mastodon/accounts.scss
@@ -325,3 +325,19 @@
     margin-top: 10px;
   }
 }
+
+.batch-table__row--muted .pending-account__header {
+  &,
+  a,
+  strong {
+    color: lighten($ui-base-color, 26%);
+  }
+}
+
+.batch-table__row--attention .pending-account__header {
+  &,
+  a,
+  strong {
+    color: $gold-star;
+  }
+}
diff --git a/app/javascript/styles/mastodon/dashboard.scss b/app/javascript/styles/mastodon/dashboard.scss
index 5e900e8c5..0a881bc10 100644
--- a/app/javascript/styles/mastodon/dashboard.scss
+++ b/app/javascript/styles/mastodon/dashboard.scss
@@ -100,6 +100,16 @@
       transition: all 200ms ease-out;
     }
 
+    &.positive {
+      background: lighten($ui-base-color, 4%);
+      color: $valid-value-color;
+    }
+
+    &.negative {
+      background: lighten($ui-base-color, 4%);
+      color: $error-value-color;
+    }
+
     span {
       flex: 1 1 auto;
     }
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index d2ec122a4..3aeecb4ec 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -129,8 +129,6 @@ class ActivityPub::Activity
   end
 
   def crawl_links(status)
-    return if status.spoiler_text?
-
     # Spread out crawling randomly to avoid DDoSing the link
     LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 9f778ffb9..6c5d88d18 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
         visibility: visibility_from_audience
       )
 
-      original_status.tags.each do |tag|
-        tag.use!(@account)
-      end
+      Trends.tags.register(@status)
+      Trends.links.register(@status)
 
       distribute(@status)
     end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 4c13a80a6..8a0dc9d33 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
-      tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
+      tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
     end
 
+    # If we're processing an old status, this may register tags as being used now
+    # as opposed to when the status was really published, but this is probably
+    # not a big deal
+    Trends.tags.register(status)
+
     @mentions.each do |mention|
       mention.status = status
       mention.save
diff --git a/app/lib/admin/metrics/dimension.rb b/app/lib/admin/metrics/dimension.rb
index 279539f68..d8392ddfc 100644
--- a/app/lib/admin/metrics/dimension.rb
+++ b/app/lib/admin/metrics/dimension.rb
@@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
     servers: Admin::Metrics::Dimension::ServersDimension,
     space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
     software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
+    tag_servers: Admin::Metrics::Dimension::TagServersDimension,
+    tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
   }.freeze
 
-  def self.retrieve(dimension_keys, start_at, end_at, limit)
-    Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact
+  def self.retrieve(dimension_keys, start_at, end_at, limit, params)
+    Array(dimension_keys).map do |key|
+      klass = DIMENSIONS[key.to_sym]
+      klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
+    end.compact
   end
 end
diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb
index 8ed8d7683..5872c22cb 100644
--- a/app/lib/admin/metrics/dimension/base_dimension.rb
+++ b/app/lib/admin/metrics/dimension/base_dimension.rb
@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::BaseDimension
-  def initialize(start_at, end_at, limit)
+  def self.with_params?
+    false
+  end
+
+  def initialize(start_at, end_at, limit, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
     @limit    = limit&.to_i
+    @params   = params
   end
 
   def key
@@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
   protected
 
   def time_period
-    (@start_at...@end_at)
+    (@start_at..@end_at)
+  end
+
+  def params
+    raise NotImplementedError
   end
 end
diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb
index 2d0ac124e..a6aaf5d21 100644
--- a/app/lib/admin/metrics/dimension/languages_dimension.rb
+++ b/app/lib/admin/metrics/dimension/languages_dimension.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
+  include LanguagesHelper
+
   def key
     'languages'
   end
@@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
 
     rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
 
-    rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } }
+    rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
   end
 end
diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
new file mode 100644
index 000000000..1cfa07478
--- /dev/null
+++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
+  include LanguagesHelper
+
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_languages'
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
+      FROM statuses
+      INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
+      WHERE statuses_tags.tag_id = $1
+        AND statuses.id BETWEEN $2 AND $3
+      GROUP BY COALESCE(statuses.language, 'und')
+      ORDER BY count(*) DESC
+      LIMIT $4
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
+
+    rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
+  end
+
+  private
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
new file mode 100644
index 000000000..12c5980d7
--- /dev/null
+++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_servers'
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT accounts.domain, count(*) AS value
+      FROM statuses
+      INNER JOIN accounts ON accounts.id = statuses.account_id
+      INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
+      WHERE statuses_tags.tag_id = $1
+        AND statuses.id BETWEEN $2 AND $3
+      GROUP BY accounts.domain
+      ORDER BY count(*) DESC
+      LIMIT $4
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
+
+    rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
+  end
+
+  private
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/measure.rb b/app/lib/admin/metrics/measure.rb
index 5cebf0331..a839498a1 100644
--- a/app/lib/admin/metrics/measure.rb
+++ b/app/lib/admin/metrics/measure.rb
@@ -7,9 +7,15 @@ class Admin::Metrics::Measure
     interactions: Admin::Metrics::Measure::InteractionsMeasure,
     opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
     resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
+    tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
+    tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
+    tag_servers: Admin::Metrics::Measure::TagServersMeasure,
   }.freeze
 
-  def self.retrieve(measure_keys, start_at, end_at)
-    Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact
+  def self.retrieve(measure_keys, start_at, end_at, params)
+    Array(measure_keys).map do |key|
+      klass = MEASURES[key.to_sym]
+      klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
+    end.compact
   end
 end
diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb
index ac022eb9d..513189780 100644
--- a/app/lib/admin/metrics/measure/active_users_measure.rb
+++ b/app/lib/admin/metrics/measure/active_users_measure.rb
@@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
   end
 
   def time_period
-    (@start_at.to_date...@end_at.to_date)
+    (@start_at.to_date..@end_at.to_date)
   end
 
   def previous_time_period
-    ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
   end
 end
diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb
index 4c336a69e..0107ffd9c 100644
--- a/app/lib/admin/metrics/measure/base_measure.rb
+++ b/app/lib/admin/metrics/measure/base_measure.rb
@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Measure::BaseMeasure
-  def initialize(start_at, end_at)
+  def self.with_params?
+    false
+  end
+
+  def initialize(start_at, end_at, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
+    @params   = params
   end
 
   def key
@@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
   protected
 
   def time_period
-    (@start_at...@end_at)
+    (@start_at..@end_at)
   end
 
   def previous_time_period
-    ((@start_at - length_of_period)...(@end_at - length_of_period))
+    ((@start_at - length_of_period)..(@end_at - length_of_period))
   end
 
   def length_of_period
     @length_of_period ||= @end_at - @start_at
   end
+
+  def params
+    raise NotImplementedError
+  end
 end
diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb
index 9a4ef6d63..b928fdb8f 100644
--- a/app/lib/admin/metrics/measure/interactions_measure.rb
+++ b/app/lib/admin/metrics/measure/interactions_measure.rb
@@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
   end
 
   def time_period
-    (@start_at.to_date...@end_at.to_date)
+    (@start_at.to_date..@end_at.to_date)
   end
 
   def previous_time_period
-    ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
   end
 end
diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb
new file mode 100644
index 000000000..ef773081b
--- /dev/null
+++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_accounts'
+  end
+
+  def total
+    tag.history.aggregate(time_period).accounts
+  end
+
+  def previous_total
+    tag.history.aggregate(previous_time_period).accounts
+  end
+
+  def data
+    time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def time_period
+    (@start_at.to_date..@end_at.to_date)
+  end
+
+  def previous_time_period
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb
new file mode 100644
index 000000000..8c3e0551a
--- /dev/null
+++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_servers'
+  end
+
+  def total
+    tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
+  end
+
+  def previous_total
+    tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT axis.*, (
+        SELECT count(*) AS value
+        FROM statuses
+        WHERE statuses.id BETWEEN $1 AND $2
+          AND date_trunc('day', statuses.created_at)::date = axis.day
+      )
+      FROM (
+        SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
+      ) as axis
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
+
+    rows.map { |row| { date: row['day'], value: row['value'].to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb
new file mode 100644
index 000000000..b7667bc6c
--- /dev/null
+++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_uses'
+  end
+
+  def total
+    tag.history.aggregate(time_period).uses
+  end
+
+  def previous_total
+    tag.history.aggregate(previous_time_period).uses
+  end
+
+  def data
+    time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def time_period
+    (@start_at.to_date..@end_at.to_date)
+  end
+
+  def previous_time_period
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index 8b38e8d0c..56ad0717b 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -4,6 +4,11 @@ class LinkDetailsExtractor
   include ActionView::Helpers::TagHelper
 
   class StructuredData
+    SUPPORTED_TYPES = %w(
+      NewsArticle
+      WebPage
+    ).freeze
+
     def initialize(data)
       @data = data
     end
@@ -16,6 +21,14 @@ class LinkDetailsExtractor
       json['description']
     end
 
+    def language
+      json['inLanguage']
+    end
+
+    def type
+      json['@type']
+    end
+
     def image
       obj = first_of_value(json['image'])
 
@@ -44,6 +57,10 @@ class LinkDetailsExtractor
       publisher['name']
     end
 
+    def publisher_logo
+      publisher.dig('logo', 'url')
+    end
+
     private
 
     def author
@@ -58,8 +75,12 @@ class LinkDetailsExtractor
       arr.is_a?(Array) ? arr.first : arr
     end
 
+    def root_array(root)
+      root.is_a?(Array) ? root : [root]
+    end
+
     def json
-      @json ||= first_of_value(Oj.load(@data))
+      @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
     end
   end
 
@@ -75,6 +96,7 @@ class LinkDetailsExtractor
       description: description || '',
       image_remote_url: image,
       type: type,
+      link_type: link_type,
       width: width || 0,
       height: height || 0,
       html: html || '',
@@ -83,6 +105,7 @@ class LinkDetailsExtractor
       author_name: author_name || '',
       author_url: author_url || '',
       embed_url: embed_url || '',
+      language: language,
     }
   end
 
@@ -90,6 +113,14 @@ class LinkDetailsExtractor
     player_url.present? ? :video : :link
   end
 
+  def link_type
+    if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
+      :article
+    else
+      :unknown
+    end
+  end
+
   def html
     player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
   end
@@ -138,6 +169,14 @@ class LinkDetailsExtractor
     valid_url_or_nil(opengraph_tag('twitter:player:stream'))
   end
 
+  def language
+    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
+  end
+
+  def icon
+    valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
+  end
+
   private
 
   def player_url
@@ -162,6 +201,14 @@ class LinkDetailsExtractor
     nil
   end
 
+  def valid_locale_or_nil(str)
+    return nil if str.blank?
+
+    code,  = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
+    locale = ISO_639.find(code)
+    locale&.alpha2
+  end
+
   def link_tag(name)
     document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
   end
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index 11fd09e30..0fbd9932d 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer
     end
   end
 
-  def new_trending_tag(recipient, tag)
-    @tag      = tag
-    @me       = recipient
-    @instance = Rails.configuration.x.local_domain
+  def new_trending_tags(recipient, tags)
+    @tags                = tags
+    @me                  = recipient
+    @instance            = Rails.configuration.x.local_domain
+    @lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
+    end
+  end
+
+  def new_trending_links(recipient, links)
+    @links                = links
+    @me                   = recipient
+    @instance             = Rails.configuration.x.local_domain
+    @lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last
 
     locale_for_account(@me) do
-      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
     end
   end
 end
diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb
index 0a9551ec2..0f78c1a54 100644
--- a/app/models/account_statuses_cleanup_policy.rb
+++ b/app/models/account_statuses_cleanup_policy.rb
@@ -4,8 +4,8 @@
 #
 # Table name: account_statuses_cleanup_policies
 #
-#  id                 :bigint           not null, primary key
-#  account_id         :bigint           not null
+#  id                 :bigint(8)        not null, primary key
+#  account_id         :bigint(8)        not null
 #  enabled            :boolean          default(TRUE), not null
 #  min_status_age     :integer          default(1209600), not null
 #  keep_direct        :boolean          default(TRUE), not null
diff --git a/app/models/form/preview_card_batch.rb b/app/models/form/preview_card_batch.rb
new file mode 100644
index 000000000..5f6e6522a
--- /dev/null
+++ b/app/models/form/preview_card_batch.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Form::PreviewCardBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :preview_card_ids, :action, :current_account, :precision
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'approve_all'
+      approve_all!
+    when 'reject'
+      reject!
+    when 'reject_all'
+      reject_all!
+    end
+  end
+
+  private
+
+  def preview_cards
+    @preview_cards ||= PreviewCard.where(id: preview_card_ids)
+  end
+
+  def preview_card_providers
+    @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
+  end
+
+  def approve!
+    preview_cards.each { |preview_card| authorize(preview_card, :update?) }
+    preview_cards.update_all(trendable: true)
+  end
+
+  def approve_all!
+    preview_card_providers.each do |provider|
+      authorize(provider, :update?)
+      provider.update(trendable: true, reviewed_at: action_time)
+    end
+
+    # Reset any individual overrides
+    preview_cards.update_all(trendable: nil)
+  end
+
+  def reject!
+    preview_cards.each { |preview_card| authorize(preview_card, :update?) }
+    preview_cards.update_all(trendable: false)
+  end
+
+  def reject_all!
+    preview_card_providers.each do |provider|
+      authorize(provider, :update?)
+      provider.update(trendable: false, reviewed_at: action_time)
+    end
+
+    # Reset any individual overrides
+    preview_cards.update_all(trendable: nil)
+  end
+
+  def action_time
+    @action_time ||= Time.now.utc
+  end
+end
diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/form/preview_card_provider_batch.rb
new file mode 100644
index 000000000..e6ab3d8fa
--- /dev/null
+++ b/app/models/form/preview_card_provider_batch.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Form::PreviewCardProviderBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :preview_card_provider_ids, :action, :current_account
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'reject'
+      reject!
+    end
+  end
+
+  private
+
+  def preview_card_providers
+    PreviewCardProvider.where(id: preview_card_provider_ids)
+  end
+
+  def approve!
+    preview_card_providers.each { |provider| authorize(provider, :update?) }
+    preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
+  end
+
+  def reject!
+    preview_card_providers.each { |provider| authorize(provider, :update?) }
+    preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
+  end
+end
diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb
index fd517a1a6..b9330745f 100644
--- a/app/models/form/tag_batch.rb
+++ b/app/models/form/tag_batch.rb
@@ -23,11 +23,15 @@ class Form::TagBatch
 
   def approve!
     tags.each { |tag| authorize(tag, :update?) }
-    tags.update_all(trendable: true, reviewed_at: Time.now.utc)
+    tags.update_all(trendable: true, reviewed_at: action_time)
   end
 
   def reject!
     tags.each { |tag| authorize(tag, :update?) }
-    tags.update_all(trendable: false, reviewed_at: Time.now.utc)
+    tags.update_all(trendable: false, reviewed_at: action_time)
+  end
+
+  def action_time
+    @action_time ||= Time.now.utc
   end
 end
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index bca3a3ce8..f2ab8ecab 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -24,6 +24,11 @@
 #  embed_url                    :string           default(""), not null
 #  image_storage_schema_version :integer
 #  blurhash                     :string
+#  language                     :string
+#  max_score                    :float
+#  max_score_at                 :datetime
+#  trendable                    :boolean
+#  link_type                    :integer
 #
 
 class PreviewCard < ApplicationRecord
@@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord
   self.inheritance_column = false
 
   enum type: [:link, :photo, :video, :rich]
+  enum link_type: [:unknown, :article]
 
   has_and_belongs_to_many :statuses
 
@@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord
 
   before_save :extract_dimensions, if: :link?
 
+  def appropriate_for_trends?
+    link? && article? && title.present? && description.present? && image.present? && provider_name.present?
+  end
+
+  def domain
+    @domain ||= Addressable::URI.parse(url).normalized_host
+  end
+
+  def provider
+    @provider ||= PreviewCardProvider.matching_domain(domain)
+  end
+
+  def trendable?
+    if attributes['trendable'].nil?
+      provider&.trendable?
+    else
+      attributes['trendable']
+    end
+  end
+
+  def requires_review_notification?
+    attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?)
+  end
+
+  attr_writer :provider
+
   def local?
     false
   end
@@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord
     save!
   end
 
+  def history
+    @history ||= Trends::History.new('links', id)
+  end
+
   class << self
     private
 
-    # rubocop:disable Naming/MethodParameterName
-    def image_styles(f)
+    def image_styles(file)
       styles = {
         original: {
           geometry: '400x400>',
@@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord
         },
       }
 
-      styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif'
+      styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif'
       styles
     end
-    # rubocop:enable Naming/MethodParameterName
   end
 
   private
diff --git a/app/models/preview_card_filter.rb b/app/models/preview_card_filter.rb
new file mode 100644
index 000000000..8dda9989c
--- /dev/null
+++ b/app/models/preview_card_filter.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class PreviewCardFilter
+  KEYS = %i(
+    trending
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = PreviewCard.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'trending'
+      trending_scope(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+
+  def trending_scope(value)
+    ids = begin
+      case value.to_s
+      when 'allowed'
+        Trends.links.currently_trending_ids(true, -1)
+      else
+        Trends.links.currently_trending_ids(false, -1)
+      end
+    end
+
+    if ids.empty?
+      PreviewCard.none
+    else
+      PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
+    end
+  end
+end
diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb
new file mode 100644
index 000000000..15b24e2bd
--- /dev/null
+++ b/app/models/preview_card_provider.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: preview_card_providers
+#
+#  id                  :bigint(8)        not null, primary key
+#  domain              :string           default(""), not null
+#  icon_file_name      :string
+#  icon_content_type   :string
+#  icon_file_size      :bigint(8)
+#  icon_updated_at     :datetime
+#  trendable           :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+
+class PreviewCardProvider < ApplicationRecord
+  include DomainNormalizable
+  include Attachmentable
+
+  ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze
+  LIMIT = 1.megabyte
+
+  validates :domain, presence: true, uniqueness: true, domain: true
+
+  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
+  validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
+  remotable_attachment :icon, LIMIT
+
+  scope :trendable, -> { where(trendable: true) }
+  scope :not_trendable, -> { where(trendable: false) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :pending_review, -> { where(reviewed_at: nil) }
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def requires_review_notification?
+    requires_review? && !requested_review?
+  end
+
+  def self.matching_domain(domain)
+    segments = domain.split('.')
+    where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first
+  end
+end
diff --git a/app/models/preview_card_provider_filter.rb b/app/models/preview_card_provider_filter.rb
new file mode 100644
index 000000000..1e90d3c9d
--- /dev/null
+++ b/app/models/preview_card_provider_filter.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class PreviewCardProviderFilter
+  KEYS = %i(
+    status
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = PreviewCardProvider.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope.order(domain: :asc)
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'status'
+      status_scope(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+
+  def status_scope(value)
+    case value.to_s
+    when 'approved'
+      PreviewCardProvider.trendable
+    when 'rejected'
+      PreviewCardProvider.not_trendable
+    when 'pending_review'
+      PreviewCardProvider.pending_review
+    else
+      raise "Unknown status: #{value}"
+    end
+  end
+end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index dcce28391..f35d92b5d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -36,6 +36,7 @@ class Tag < ApplicationRecord
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
+  scope :not_trendable, -> { where(trendable: false) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
   scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
@@ -75,28 +76,12 @@ class Tag < ApplicationRecord
     requested_review_at.present?
   end
 
-  def use!(account, status: nil, at_time: Time.now.utc)
-    TrendingTags.record_use!(self, account, status: status, at_time: at_time)
-  end
-
-  def trending?
-    TrendingTags.trending?(self)
+  def requires_review_notification?
+    requires_review? && !requested_review?
   end
 
   def history
-    days = []
-
-    7.times do |i|
-      day = i.days.ago.beginning_of_day.to_i
-
-      days << {
-        day: day.to_s,
-        uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
-        accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
-      }
-    end
-
-    days
+    @history ||= Trends::History.new('tags', id)
   end
 
   class << self
diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb
index 85bfcbea5..ecdb52503 100644
--- a/app/models/tag_filter.rb
+++ b/app/models/tag_filter.rb
@@ -2,13 +2,8 @@
 
 class TagFilter
   KEYS = %i(
-    directory
-    reviewed
-    unreviewed
-    pending_review
-    popular
-    active
-    name
+    trending
+    status
   ).freeze
 
   attr_reader :params
@@ -18,7 +13,13 @@ class TagFilter
   end
 
   def results
-    scope = Tag.unscoped
+    scope = begin
+      if params[:status] == 'pending_review'
+        Tag.unscoped
+      else
+        trending_scope
+      end
+    end
 
     params.each do |key, value|
       next if key.to_s == 'page'
@@ -26,27 +27,40 @@ class TagFilter
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end
 
-    scope.order(id: :desc)
+    scope
   end
 
   private
 
   def scope_for(key, value)
     case key.to_s
-    when 'reviewed'
-      Tag.reviewed.order(reviewed_at: :desc)
-    when 'unreviewed'
-      Tag.unreviewed
-    when 'pending_review'
-      Tag.pending_review.order(requested_review_at: :desc)
-    when 'popular'
-      Tag.order('max_score DESC NULLS LAST')
-    when 'active'
-      Tag.order('last_status_at DESC NULLS LAST')
-    when 'name'
-      Tag.matches_name(value)
+    when 'status'
+      status_scope(value)
     else
       raise "Unknown filter: #{key}"
     end
   end
+
+  def trending_scope
+    ids = Trends.tags.currently_trending_ids(false, -1)
+
+    if ids.empty?
+      Tag.none
+    else
+      Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
+    end
+  end
+
+  def status_scope(value)
+    case value.to_s
+    when 'approved'
+      Tag.trendable
+    when 'rejected'
+      Tag.not_trendable
+    when 'pending_review'
+      Tag.pending_review
+    else
+      raise "Unknown status: #{value}"
+    end
+  end
 end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
deleted file mode 100644
index 31890b082..000000000
--- a/app/models/trending_tags.rb
+++ /dev/null
@@ -1,128 +0,0 @@
-# frozen_string_literal: true
-
-class TrendingTags
-  KEY                  = 'trending_tags'
-  EXPIRE_HISTORY_AFTER = 7.days.seconds
-  EXPIRE_TRENDS_AFTER  = 1.day.seconds
-  THRESHOLD            = 5
-  LIMIT                = 10
-  REVIEW_THRESHOLD     = 3
-  MAX_SCORE_COOLDOWN   = 2.days.freeze
-  MAX_SCORE_HALFLIFE   = 2.hours.freeze
-
-  class << self
-    include Redisable
-
-    def record_use!(tag, account, status: nil, at_time: Time.now.utc)
-      return unless tag.usable? && !account.silenced?
-
-      # Even if a tag is not allowed to trend, we still need to
-      # record the stats since they can be displayed in other places
-      increment_historical_use!(tag.id, at_time)
-      increment_unique_use!(tag.id, account.id, at_time)
-      increment_use!(tag.id, at_time)
-
-      # Only update when the tag was last used once every 12 hours
-      # and only if a status is given (lets use ignore reblogs)
-      tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
-    end
-
-    def update!(at_time = Time.now.utc)
-      tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
-      tags    = Tag.trendable.where(id: tag_ids.uniq)
-
-      # First pass to calculate scores and update the set
-
-      tags.each do |tag|
-        expected  = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
-        expected  = 1.0 if expected.zero?
-        observed  = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
-        max_time  = tag.max_score_at
-        max_score = tag.max_score
-        max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
-
-        score = begin
-          if expected > observed || observed < THRESHOLD
-            0
-          else
-            ((observed - expected)**2) / expected
-          end
-        end
-
-        if score > max_score
-          max_score = score
-          max_time  = at_time
-
-          # Not interested in triggering any callbacks for this
-          tag.update_columns(max_score: max_score, max_score_at: max_time)
-        end
-
-        decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
-
-        if decaying_score.zero?
-          redis.zrem(KEY, tag.id)
-        else
-          redis.zadd(KEY, decaying_score, tag.id)
-        end
-      end
-
-      users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
-
-      # Second pass to notify about previously unreviewed trends
-
-      tags.each do |tag|
-        current_rank              = redis.zrevrank(KEY, tag.id)
-        needs_review_notification = tag.requires_review? && !tag.requested_review?
-        rank_passes_threshold     = current_rank.present? && current_rank <= REVIEW_THRESHOLD
-
-        next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
-
-        tag.touch(:requested_review_at)
-
-        users_for_review.each do |user|
-          AdminMailer.new_trending_tag(user.account, tag).deliver_later!
-        end
-      end
-
-      # Trim older items
-
-      redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
-      redis.zremrangebyscore(KEY, '(0.3', '-inf')
-    end
-
-    def get(limit, filtered: true)
-      tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
-
-      tags = Tag.where(id: tag_ids)
-      tags = tags.trendable if filtered
-      tags = tags.index_by(&:id)
-
-      tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
-    end
-
-    def trending?(tag)
-      rank = redis.zrevrank(KEY, tag.id)
-      rank.present? && rank < LIMIT
-    end
-
-    private
-
-    def increment_historical_use!(tag_id, at_time)
-      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
-      redis.incrby(key, 1)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-
-    def increment_unique_use!(tag_id, account_id, at_time)
-      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
-      redis.pfadd(key, account_id)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-
-    def increment_use!(tag_id, at_time)
-      key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
-      redis.sadd(key, tag_id)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-  end
-end
diff --git a/app/models/trends.rb b/app/models/trends.rb
new file mode 100644
index 000000000..7dd3a9c87
--- /dev/null
+++ b/app/models/trends.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Trends
+  def self.table_name_prefix
+    'trends_'
+  end
+
+  def self.links
+    @links ||= Trends::Links.new
+  end
+
+  def self.tags
+    @tags ||= Trends::Tags.new
+  end
+
+  def self.refresh!
+    [links, tags].each(&:refresh)
+  end
+
+  def self.request_review!
+    [links, tags].each(&:request_review) if enabled?
+  end
+
+  def self.enabled?
+    Setting.trends
+  end
+end
diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb
new file mode 100644
index 000000000..b767dcb1a
--- /dev/null
+++ b/app/models/trends/base.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class Trends::Base
+  include Redisable
+
+  class_attribute :default_options
+
+  attr_reader :options
+
+  # @param [Hash] options
+  # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score
+  # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review
+  # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from
+  # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays
+  def initialize(options = {})
+    @options = self.class.default_options.merge(options)
+  end
+
+  def register(_status)
+    raise NotImplementedError
+  end
+
+  def add(*)
+    raise NotImplementedError
+  end
+
+  def refresh(*)
+    raise NotImplementedError
+  end
+
+  def request_review
+    raise NotImplementedError
+  end
+
+  def get(*)
+    raise NotImplementedError
+  end
+
+  def score(id)
+    redis.zscore("#{key_prefix}:all", id) || 0
+  end
+
+  def rank(id)
+    redis.zrevrank("#{key_prefix}:allowed", id)
+  end
+
+  def currently_trending_ids(allowed, limit)
+    redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
+  end
+
+  protected
+
+  def key_prefix
+    raise NotImplementedError
+  end
+
+  def recently_used_ids(at_time = Time.now.utc)
+    redis.smembers(used_key(at_time)).map(&:to_i)
+  end
+
+  def record_used_id(id, at_time = Time.now.utc)
+    redis.sadd(used_key(at_time), id)
+    redis.expire(used_key(at_time), 1.day.seconds)
+  end
+
+  def trim_older_items
+    redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
+    redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
+  end
+
+  def score_at_rank(rank)
+    redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
+  end
+
+  private
+
+  def used_key(at_time)
+    "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
+  end
+end
diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb
new file mode 100644
index 000000000..608e33792
--- /dev/null
+++ b/app/models/trends/history.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+class Trends::History
+  include Enumerable
+
+  class Aggregate
+    include Redisable
+
+    def initialize(prefix, id, date_range)
+      @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) }
+    end
+
+    def uses
+      redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum
+    end
+
+    def accounts
+      redis.pfcount(*@days.map { |day| day.key_for(:accounts) })
+    end
+  end
+
+  class Day
+    include Redisable
+
+    EXPIRE_AFTER = 14.days.seconds
+
+    def initialize(prefix, id, day)
+      @prefix = prefix
+      @id     = id
+      @day    = day.beginning_of_day
+    end
+
+    attr_reader :day
+
+    def accounts
+      redis.pfcount(key_for(:accounts))
+    end
+
+    def uses
+      redis.get(key_for(:uses))&.to_i || 0
+    end
+
+    def add(account_id)
+      redis.pipelined do
+        redis.incrby(key_for(:uses), 1)
+        redis.pfadd(key_for(:accounts), account_id)
+        redis.expire(key_for(:uses), EXPIRE_AFTER)
+        redis.expire(key_for(:accounts), EXPIRE_AFTER)
+      end
+    end
+
+    def as_json
+      { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s }
+    end
+
+    def key_for(suffix)
+      case suffix
+      when :accounts
+        "#{key_prefix}:#{suffix}"
+      when :uses
+        key_prefix
+      end
+    end
+
+    def key_prefix
+      "activity:#{@prefix}:#{@id}:#{day.to_i}"
+    end
+  end
+
+  def initialize(prefix, id)
+    @prefix = prefix
+    @id     = id
+  end
+
+  def get(date)
+    Day.new(@prefix, @id, date)
+  end
+
+  def add(account_id, at_time = Time.now.utc)
+    Day.new(@prefix, @id, at_time).add(account_id)
+  end
+
+  def aggregate(date_range)
+    Aggregate.new(@prefix, @id, date_range)
+  end
+
+  def each(&block)
+    if block_given?
+      (0...7).map { |i| block.call(get(i.days.ago)) }
+    else
+      to_enum(:each)
+    end
+  end
+
+  def as_json(*)
+    map(&:as_json)
+  end
+end
diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb
new file mode 100644
index 000000000..a0d65138b
--- /dev/null
+++ b/app/models/trends/links.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+class Trends::Links < Trends::Base
+  PREFIX = 'trending_links'
+
+  self.default_options = {
+    threshold: 15,
+    review_threshold: 10,
+    max_score_cooldown: 2.days.freeze,
+    max_score_halflife: 8.hours.freeze,
+  }
+
+  def register(status, at_time = Time.now.utc)
+    original_status = status.reblog? ? status.reblog : status
+
+    return unless original_status.public_visibility? && status.public_visibility? &&
+                  !original_status.account.silenced? && !status.account.silenced? &&
+                  !original_status.spoiler_text?
+
+    original_status.preview_cards.each do |preview_card|
+      add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends?
+    end
+  end
+
+  def add(preview_card, account_id, at_time = Time.now.utc)
+    preview_card.history.add(account_id, at_time)
+    record_used_id(preview_card.id, at_time)
+  end
+
+  def get(allowed, limit)
+    preview_card_ids = currently_trending_ids(allowed, limit)
+    preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
+    preview_card_ids.map { |id| preview_cards[id] }.compact
+  end
+
+  def refresh(at_time = Time.now.utc)
+    preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    calculate_scores(preview_cards, at_time)
+    trim_older_items
+  end
+
+  def request_review
+    preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
+
+    preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
+      next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
+
+      if preview_card.provider.nil?
+        preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
+      else
+        preview_card.provider.touch(:requested_review_at)
+      end
+
+      preview_card
+    end
+
+    return if preview_cards_requiring_review.empty?
+
+    User.staff.includes(:account).find_each do |user|
+      AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
+    end
+  end
+
+  protected
+
+  def key_prefix
+    PREFIX
+  end
+
+  private
+
+  def calculate_scores(preview_cards, at_time)
+    preview_cards.each do |preview_card|
+      expected  = preview_card.history.get(at_time - 1.day).accounts.to_f
+      expected  = 1.0 if expected.zero?
+      observed  = preview_card.history.get(at_time).accounts.to_f
+      max_time  = preview_card.max_score_at
+      max_score = preview_card.max_score
+      max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
+
+      score = begin
+        if expected > observed || observed < options[:threshold]
+          0
+        else
+          ((observed - expected)**2) / expected
+        end
+      end
+
+      if score > max_score
+        max_score = score
+        max_time  = at_time
+
+        # Not interested in triggering any callbacks for this
+        preview_card.update_columns(max_score: max_score, max_score_at: max_time)
+      end
+
+      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+
+      if decaying_score.zero?
+        redis.zrem("#{PREFIX}:all", preview_card.id)
+        redis.zrem("#{PREFIX}:allowed", preview_card.id)
+      else
+        redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
+
+        if preview_card.trendable?
+          redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
+        else
+          redis.zrem("#{PREFIX}:allowed", preview_card.id)
+        end
+      end
+    end
+  end
+
+  def would_be_trending?(id)
+    score(id) > score_at_rank(options[:review_threshold] - 1)
+  end
+end
diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb
new file mode 100644
index 000000000..13e0ab56b
--- /dev/null
+++ b/app/models/trends/tags.rb
@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+class Trends::Tags < Trends::Base
+  PREFIX = 'trending_tags'
+
+  self.default_options = {
+    threshold: 15,
+    review_threshold: 10,
+    max_score_cooldown: 2.days.freeze,
+    max_score_halflife: 4.hours.freeze,
+  }
+
+  def register(status, at_time = Time.now.utc)
+    original_status = status.reblog? ? status.reblog : status
+
+    return unless original_status.public_visibility? && status.public_visibility? &&
+                  !original_status.account.silenced? && !status.account.silenced?
+
+    original_status.tags.each do |tag|
+      add(tag, status.account_id, at_time) if tag.usable?
+    end
+  end
+
+  def add(tag, account_id, at_time = Time.now.utc)
+    tag.history.add(account_id, at_time)
+    record_used_id(tag.id, at_time)
+  end
+
+  def refresh(at_time = Time.now.utc)
+    tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    calculate_scores(tags, at_time)
+    trim_older_items
+  end
+
+  def get(allowed, limit)
+    tag_ids = currently_trending_ids(allowed, limit)
+    tags = Tag.where(id: tag_ids).index_by(&:id)
+    tag_ids.map { |id| tags[id] }.compact
+  end
+
+  def request_review
+    tags = Tag.where(id: currently_trending_ids(false, -1))
+
+    tags_requiring_review = tags.filter_map do |tag|
+      next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
+
+      tag.touch(:requested_review_at)
+      tag
+    end
+
+    return if tags_requiring_review.empty?
+
+    User.staff.includes(:account).find_each do |user|
+      AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
+    end
+  end
+
+  protected
+
+  def key_prefix
+    PREFIX
+  end
+
+  private
+
+  def calculate_scores(tags, at_time)
+    tags.each do |tag|
+      expected  = tag.history.get(at_time - 1.day).accounts.to_f
+      expected  = 1.0 if expected.zero?
+      observed  = tag.history.get(at_time).accounts.to_f
+      max_time  = tag.max_score_at
+      max_score = tag.max_score
+      max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
+
+      score = begin
+        if expected > observed || observed < options[:threshold]
+          0
+        else
+          ((observed - expected)**2) / expected
+        end
+      end
+
+      if score > max_score
+        max_score = score
+        max_time  = at_time
+
+        # Not interested in triggering any callbacks for this
+        tag.update_columns(max_score: max_score, max_score_at: max_time)
+      end
+
+      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+
+      if decaying_score.zero?
+        redis.zrem("#{PREFIX}:all", tag.id)
+        redis.zrem("#{PREFIX}:allowed", tag.id)
+      else
+        redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
+
+        if tag.trendable?
+          redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
+        else
+          redis.zrem("#{PREFIX}:allowed", tag.id)
+        end
+      end
+    end
+  end
+
+  def would_be_trending?(id)
+    score(id) > score_at_rank(options[:review_threshold] - 1)
+  end
+end
diff --git a/app/policies/preview_card_policy.rb b/app/policies/preview_card_policy.rb
new file mode 100644
index 000000000..4f485d7fc
--- /dev/null
+++ b/app/policies/preview_card_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class PreviewCardPolicy < ApplicationPolicy
+  def index?
+    staff?
+  end
+
+  def update?
+    staff?
+  end
+end
diff --git a/app/policies/preview_card_provider_policy.rb b/app/policies/preview_card_provider_policy.rb
new file mode 100644
index 000000000..598d54a5e
--- /dev/null
+++ b/app/policies/preview_card_provider_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class PreviewCardProviderPolicy < ApplicationPolicy
+  def index?
+    staff?
+  end
+
+  def update?
+    staff?
+  end
+end
diff --git a/app/serializers/rest/trends/link_serializer.rb b/app/serializers/rest/trends/link_serializer.rb
new file mode 100644
index 000000000..232483490
--- /dev/null
+++ b/app/serializers/rest/trends/link_serializer.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class REST::Trends::LinkSerializer < REST::PreviewCardSerializer
+  attributes :history
+end
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 51956ce7e..94dc6389f 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -50,7 +50,7 @@ class FetchLinkCardService < BaseService
       # We follow redirects, and ideally we want to save the preview card for
       # the destination URL and not any link shortener in-between, so here
       # we set the URL to the one of the last response in the redirect chain
-      @url  = res.request.uri.to_s.to_s
+      @url  = res.request.uri.to_s
       @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url
 
       if res.code == 200 && res.mime_type == 'text/html'
@@ -66,6 +66,7 @@ class FetchLinkCardService < BaseService
   def attach_card
     @status.preview_cards << @card
     Rails.cache.delete(@status)
+    Trends.links.register(@status)
   end
 
   def parse_urls
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 85aaec4d6..294ae43eb 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -91,7 +91,8 @@ class PostStatusService < BaseService
   end
 
   def postprocess_status!
-    LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
+    Trends.tags.register(@status)
+    LinkCrawlWorker.perform_async(@status.id)
     DistributionWorker.perform_async(@status.id)
     ActivityPub::DistributionWorker.perform_async(@status.id)
     PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index c42b79db8..47277c56c 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -8,7 +8,7 @@ class ProcessHashtagsService < BaseService
     Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
-      tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility?
+      tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
     end
 
     return unless status.distributable?
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 744bdf567..ece91847a 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -30,12 +30,13 @@ class ReblogService < BaseService
 
     reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
 
+    Trends.tags.register(reblog)
+    Trends.links.register(reblog)
     DistributionWorker.perform_async(reblog.id)
     ActivityPub::DistributionWorker.perform_async(reblog.id)
 
     create_notification(reblog)
     bump_potential_friendship(account, reblog)
-    record_use(account, reblog)
 
     reblog
   end
@@ -60,16 +61,6 @@ class ReblogService < BaseService
     PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
   end
 
-  def record_use(account, reblog)
-    return unless reblog.public_visibility?
-
-    original_status = reblog.reblog
-
-    original_status.tags.each do |tag|
-      tag.use!(account)
-    end
-  end
-
   def build_json(reblog)
     Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
   end
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index 560eba7b4..895333a58 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -42,7 +42,7 @@
       %span= t('admin.dashboard.pending_users_html', count: @pending_users_count)
       = fa_icon 'chevron-right fw'
 
-    = link_to admin_tags_path(pending_review: '1'), class: 'dashboard__quick-access' do
+    = link_to admin_trends_tags_path(status: 'pending_review'), class: 'dashboard__quick-access' do
       %span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count)
       = fa_icon 'chevron-right fw'
 
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
deleted file mode 100644
index ac0c72816..000000000
--- a/app/views/admin/tags/_tag.html.haml
+++ /dev/null
@@ -1,19 +0,0 @@
-.batch-table__row
-  - if batch_available
-    %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
-      = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
-
-  .directory__tag
-    = link_to admin_tag_path(tag.id) do
-      %h4
-        = fa_icon 'hashtag'
-        = tag.name
-
-        %small
-          = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
-
-          - if tag.trending?
-            = fa_icon 'fire fw'
-            = t('admin.tags.trending_right_now')
-
-      .trends__item__current= friendly_number_to_human tag.history.first[:uses]
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
deleted file mode 100644
index d78f3c6d1..000000000
--- a/app/views/admin/tags/index.html.haml
+++ /dev/null
@@ -1,74 +0,0 @@
-- content_for :page_title do
-  = t('admin.tags.title')
-
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
-.filters
-  .filter-subset
-    %strong= t('admin.tags.review')
-    %ul
-      %li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil
-      %li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil
-      %li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil
-      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil
-
-  .filter-subset
-    %strong= t('generic.order_by')
-    %ul
-      %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
-      %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
-      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
-
-
-= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - TagFilter::KEYS.each do |key|
-      = hidden_field_tag key, params[key] if params[key].present?
-
-    - %i(name).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}")
-
-    .actions
-      %button.button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative'
-
-%hr.spacer/
-
-= form_for(@form, url: batch_admin_tags_path) do |f|
-  = hidden_field_tag :page, params[:page] || 1
-
-  - TagFilter::KEYS.each do |key|
-    = hidden_field_tag key, params[key] if params[key].present?
-
-  .batch-table.optional
-    .batch-table__toolbar
-      - if params[:pending_review] == '1' || params[:unreviewed] == '1'
-        %label.batch-table__toolbar__select.batch-checkbox-all
-          = check_box_tag :batch_checkbox_all, nil, false
-        .batch-table__toolbar__actions
-          = f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-
-          = f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-      - else
-        .batch-table__toolbar__actions
-          %span.neutral-hint= t('generic.no_batch_actions_available')
-
-    .batch-table__body
-      - if @tags.empty?
-        = nothing_here 'nothing-here--under-tabs'
-      - else
-        = render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' }
-
-= paginate @tags
-
-- if params[:pending_review] == '1' || params[:unreviewed] == '1'
-  %hr.spacer/
-
-  %div.action-buttons
-    %div
-      = link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button'
-
-    %div
-      = link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive'
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
index c4caffda1..007dc005e 100644
--- a/app/views/admin/tags/show.html.haml
+++ b/app/views/admin/tags/show.html.haml
@@ -1,15 +1,50 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
+
 - content_for :page_title do
   = "##{@tag.name}"
 
-.dashboard__counters
-  %div
-    = link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do
-      .dashboard__counters__num= number_with_delimiter @accounts_today
-      .dashboard__counters__label= t 'admin.tags.accounts_today'
-  %div
-    %div
-      .dashboard__counters__num= number_with_delimiter @accounts_week
-      .dashboard__counters__label= t 'admin.tags.accounts_week'
+- content_for :heading_actions do
+  = l(@time_period.first)
+  = ' - '
+  = l(@time_period.last)
+
+.dashboard
+  .dashboard__item
+    = react_admin_component :counter, measure: 'tag_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_accounts_measure')
+  .dashboard__item
+    = react_admin_component :counter, measure: 'tag_uses', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_uses_measure')
+  .dashboard__item
+    = react_admin_component :counter, measure: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_servers_measure')
+  .dashboard__item
+    = react_admin_component :dimension, dimension: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_servers_dimension')
+  .dashboard__item
+    = react_admin_component :dimension, dimension: 'tag_languages', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_languages_dimension')
+  .dashboard__item
+    = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.usable? ? 'positive' : 'negative'] do
+      - if @tag.usable?
+        %span= t('admin.trends.tags.usable')
+        = fa_icon 'check fw'
+      - else
+        %span= t('admin.trends.tags.not_usable')
+        = fa_icon 'lock fw'
+
+    = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.trendable? ? 'positive' : 'negative'] do
+      - if @tag.trendable?
+        %span= t('admin.trends.tags.trendable')
+        = fa_icon 'check fw'
+      - else
+        %span= t('admin.trends.tags.not_trendable')
+        = fa_icon 'lock fw'
+
+
+    = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.listable? ? 'positive' : 'negative'] do
+      - if @tag.listable?
+        %span= t('admin.trends.tags.listable')
+        = fa_icon 'check fw'
+      - else
+        %span= t('admin.trends.tags.not_listable')
+        = fa_icon 'lock fw'
 
 %hr.spacer/
 
@@ -26,18 +61,3 @@
 
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit
-
-%hr.spacer/
-
-%h3= t 'admin.tags.breakdown'
-
-.table-wrapper
-  %table.table
-    %tbody
-      - total = @usage_by_domain.sum(&:last).to_f
-
-      - @usage_by_domain.each do |(domain, count)|
-        %tr
-          %th= domain || site_hostname
-          %td= number_to_percentage((count / total) * 100, precision: 1)
-          %td= number_with_delimiter count
diff --git a/app/views/admin/trends/links/_preview_card.html.haml b/app/views/admin/trends/links/_preview_card.html.haml
new file mode 100644
index 000000000..dfed13b68
--- /dev/null
+++ b/app/views/admin/trends/links/_preview_card.html.haml
@@ -0,0 +1,30 @@
+.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id
+
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      = link_to preview_card.title, preview_card.url
+
+      %br/
+
+      - if preview_card.provider_name.present?
+        = preview_card.provider_name
+        •
+
+      - if preview_card.language.present?
+        = human_locale(preview_card.language)
+        •
+
+      = t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts })
+
+      - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id))
+        •
+        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+
+        - if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago
+          •
+          = t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short))
+      - elsif preview_card.provider&.requires_review?
+        •
+        = t('admin.trends.pending_review')
diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml
new file mode 100644
index 000000000..240ae722b
--- /dev/null
+++ b/app/views/admin/trends/links/index.html.haml
@@ -0,0 +1,41 @@
+- content_for :page_title do
+  = t('admin.trends.links.title')
+
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
+
+.filters
+  .filter-subset
+    %strong= t('admin.trends.trending')
+    %ul
+      %li= filter_link_to t('generic.all'), trending: nil
+      %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
+  .back-link
+    = link_to admin_trends_links_preview_card_providers_path do
+      = t('admin.trends.preview_card_providers.title')
+      = fa_icon 'chevron-right fw'
+
+%hr.spacer/
+
+= form_for(@form, url: batch_admin_trends_links_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - PreviewCardFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+    .batch-table__body
+      - if @preview_cards.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'preview_card', collection: @preview_cards, locals: { f: f }
+
+= paginate @preview_cards
diff --git a/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml b/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml
new file mode 100644
index 000000000..e40e6529d
--- /dev/null
+++ b/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml
@@ -0,0 +1,16 @@
+.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id
+
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      %strong= preview_card_provider.domain
+
+      %br/
+
+      - if preview_card_provider.requires_review?
+        = t('admin.trends.pending_review')
+      - elsif preview_card_provider.trendable?
+        = t('admin.trends.preview_card_providers.allowed')
+      - else
+        = t('admin.trends.preview_card_providers.rejected')
diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml
new file mode 100644
index 000000000..eac6e641f
--- /dev/null
+++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml
@@ -0,0 +1,43 @@
+- content_for :page_title do
+  = t('admin.trends.preview_card_providers.title')
+
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
+
+.filters
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), status: nil
+      %li= filter_link_to t('admin.trends.approved'), status: 'approved'
+      %li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review'
+  .back-link
+    = link_to admin_trends_links_path do
+      = fa_icon 'chevron-left fw'
+      = t('admin.trends.links.title')
+
+
+%hr.spacer/
+
+= form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - PreviewCardProviderFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table.optional
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+    .batch-table__body
+      - if @preview_card_providers.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f }
+
+= paginate @preview_card_providers
diff --git a/app/views/admin/trends/tags/_tag.html.haml b/app/views/admin/trends/tags/_tag.html.haml
new file mode 100644
index 000000000..c4af77b00
--- /dev/null
+++ b/app/views/admin/trends/tags/_tag.html.haml
@@ -0,0 +1,24 @@
+.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
+
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      = link_to admin_tag_path(tag.id) do
+        = fa_icon 'hashtag'
+        = tag.name
+
+      %br/
+
+      = t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts })
+
+      - if tag.trendable? && (rank = Trends.tags.rank(tag.id))
+        •
+        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+
+        - if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago
+          •
+          = t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short))
+      - elsif tag.requires_review?
+        •
+        = t('admin.trends.pending_review')
diff --git a/app/views/admin/trends/tags/index.html.haml b/app/views/admin/trends/tags/index.html.haml
new file mode 100644
index 000000000..8df0a9920
--- /dev/null
+++ b/app/views/admin/trends/tags/index.html.haml
@@ -0,0 +1,38 @@
+- content_for :page_title do
+  = t('admin.trends.tags.title')
+
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
+
+.filters
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), status: nil
+      %li= filter_link_to t('admin.trends.approved'), status: 'approved'
+      %li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review'
+
+%hr.spacer/
+
+= form_for(@form, url: batch_admin_trends_tags_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - TagFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table.optional
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+    .batch-table__body
+      - if @tags.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'tag', collection: @tags, locals: { f: f }
+
+= paginate @tags
diff --git a/app/views/admin_mailer/new_trending_links.text.erb b/app/views/admin_mailer/new_trending_links.text.erb
new file mode 100644
index 000000000..51789aca5
--- /dev/null
+++ b/app/views/admin_mailer/new_trending_links.text.erb
@@ -0,0 +1,16 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_links.body') %>
+
+<% @links.each do |link| %>
+- <%= link.title %> • <%= link.url %>
+  <%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
+<% end %>
+
+<% if @lowest_trending_link %>
+<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %>
+<% else %>
+<%= t('admin_mailer.new_trending_links.no_approved_links') %>
+<% end %>
+
+<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>
diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb
deleted file mode 100644
index e4bfdc591..000000000
--- a/app/views/admin_mailer/new_trending_tag.text.erb
+++ /dev/null
@@ -1,5 +0,0 @@
-<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
-
-<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
-
-<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %>
diff --git a/app/views/admin_mailer/new_trending_tags.text.erb b/app/views/admin_mailer/new_trending_tags.text.erb
new file mode 100644
index 000000000..5051e8a96
--- /dev/null
+++ b/app/views/admin_mailer/new_trending_tags.text.erb
@@ -0,0 +1,16 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tags.body') %>
+
+<% @tags.each do |tag| %>
+- #<%= tag.name %>
+  <%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %>
+<% end %>
+
+<% if @lowest_trending_tag %>
+<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %>
+<% else %>
+<%= t('admin_mailer.new_trending_tags.no_approved_tags') %>
+<% end %>
+
+<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %>
diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml
index 7ec91c06a..6826c3b58 100644
--- a/app/views/application/_sidebar.html.haml
+++ b/app/views/application/_sidebar.html.haml
@@ -6,7 +6,7 @@
     %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html')
 
 - if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
-  - trends = TrendingTags.get(3)
+  - trends = Trends.tags.get(true, 3)
 
   - unless trends.empty?
     .endorsements-widget.trends-widget
diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb
deleted file mode 100644
index 94d76d010..000000000
--- a/app/workers/scheduler/trending_tags_scheduler.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::TrendingTagsScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options retry: 0
-
-  def perform
-    TrendingTags.update! if Setting.trends
-  end
-end
diff --git a/app/workers/scheduler/trends/refresh_scheduler.rb b/app/workers/scheduler/trends/refresh_scheduler.rb
new file mode 100644
index 000000000..b559ba46b
--- /dev/null
+++ b/app/workers/scheduler/trends/refresh_scheduler.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::Trends::RefreshScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 0
+
+  def perform
+    Trends.refresh!
+  end
+end
diff --git a/app/workers/scheduler/trends/review_notifications_scheduler.rb b/app/workers/scheduler/trends/review_notifications_scheduler.rb
new file mode 100644
index 000000000..f334261bd
--- /dev/null
+++ b/app/workers/scheduler/trends/review_notifications_scheduler.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::Trends::ReviewNotificationsScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 0
+
+  def perform
+    Trends.request_review!
+  end
+end
diff --git a/config/brakeman.ignore b/config/brakeman.ignore
index 35f2c3178..c032e5412 100644
--- a/config/brakeman.ignore
+++ b/config/brakeman.ignore
@@ -67,7 +67,7 @@
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/account.rb",
-      "line": 479,
+      "line": 484,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
       "code": "find_by_sql([\"          WITH first_degree AS (\\n            SELECT target_account_id\\n            FROM follows\\n            WHERE account_id = ?\\n            UNION ALL\\n            SELECT ?\\n          )\\n          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n          WHERE accounts.id IN (SELECT * FROM first_degree)\\n            AND #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
       "render_path": null,
@@ -100,6 +100,26 @@
       "confidence": "Weak",
       "note": ""
     },
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "75fcd147b7611763ab6915faf8c5b0709e612b460f27c05c72d8b9bd0a6a77f8",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "lib/mastodon/snowflake.rb",
+      "line": 87,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "connection.execute(\"CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\nRETURNS bigint AS\\n$$\\n  DECLARE\\n    time_part bigint;\\n    sequence_base bigint;\\n    tail bigint;\\n  BEGIN\\n    time_part := (\\n      -- Get the time in milliseconds\\n      ((date_part('epoch', now()) * 1000))::bigint\\n      -- And shift it over two bytes\\n      << 16);\\n\\n    sequence_base := (\\n      'x' ||\\n      -- Take the first two bytes (four hex characters)\\n      substr(\\n        -- Of the MD5 hash of the data we documented\\n        md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),\\n        1, 4\\n      )\\n    -- And turn it into a bigint\\n    )::bit(16)::bigint;\\n\\n    -- Finally, add our sequence number to our base, and chop\\n    -- it to the last two bytes\\n    tail := (\\n      (sequence_base + nextval(table_name || '_id_seq'))\\n      & 65535);\\n\\n    -- Return the time part and the sequence part. OR appears\\n    -- faster here than addition, but they're equivalent:\\n    -- time_part has no trailing two bytes, and tail is only\\n    -- the last two bytes.\\n    RETURN time_part | tail;\\n  END\\n$$ LANGUAGE plpgsql VOLATILE;\\n\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "Mastodon::Snowflake",
+        "method": "define_timestamp_id"
+      },
+      "user_input": "SecureRandom.hex(16)",
+      "confidence": "Medium",
+      "note": ""
+    },
     {
       "warning_type": "Mass Assignment",
       "warning_code": 105,
@@ -143,40 +163,40 @@
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "9251d682c4e2840e1b2fea91e7d758efe2097ecb7f6255c065e3750d25eb178c",
+      "fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd",
       "check_name": "SQL",
       "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 448,
+      "file": "app/models/preview_card_filter.rb",
+      "line": 50,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"        SELECT\\n          accounts.*,\\n          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n        FROM accounts\\n        WHERE #{query} @@ #{textsearch}\\n          AND accounts.suspended_at IS NULL\\n          AND accounts.moved_to_account_id IS NULL\\n        ORDER BY rank DESC\\n        LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
+      "code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")",
       "render_path": null,
       "location": {
         "type": "method",
-        "class": "Account",
-        "method": "search_for"
+        "class": "PreviewCardFilter",
+        "method": "trending_scope"
       },
-      "user_input": "textsearch",
+      "user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")",
       "confidence": "Medium",
       "note": ""
     },
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3",
+      "fingerprint": "9251d682c4e2840e1b2fea91e7d758efe2097ecb7f6255c065e3750d25eb178c",
       "check_name": "SQL",
       "message": "Possible SQL injection",
-      "file": "lib/mastodon/snowflake.rb",
-      "line": 87,
+      "file": "app/models/account.rb",
+      "line": 453,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "connection.execute(\"        CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n        RETURNS bigint AS\\n        $$\\n          DECLARE\\n            time_part bigint;\\n            sequence_base bigint;\\n            tail bigint;\\n          BEGIN\\n            time_part := (\\n              -- Get the time in milliseconds\\n              ((date_part('epoch', now()) * 1000))::bigint\\n              -- And shift it over two bytes\\n              << 16);\\n\\n            sequence_base := (\\n              'x' ||\\n              -- Take the first two bytes (four hex characters)\\n              substr(\\n                -- Of the MD5 hash of the data we documented\\n                md5(table_name ||\\n                  '#{SecureRandom.hex(16)}' ||\\n                  time_part::text\\n                ),\\n                1, 4\\n              )\\n            -- And turn it into a bigint\\n            )::bit(16)::bigint;\\n\\n            -- Finally, add our sequence number to our base, and chop\\n            -- it to the last two bytes\\n            tail := (\\n              (sequence_base + nextval(table_name || '_id_seq'))\\n              & 65535);\\n\\n            -- Return the time part and the sequence part. OR appears\\n            -- faster here than addition, but they're equivalent:\\n            -- time_part has no trailing two bytes, and tail is only\\n            -- the last two bytes.\\n            RETURN time_part | tail;\\n          END\\n        $$ LANGUAGE plpgsql VOLATILE;\\n\")",
+      "code": "find_by_sql([\"        SELECT\\n          accounts.*,\\n          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n        FROM accounts\\n        WHERE #{query} @@ #{textsearch}\\n          AND accounts.suspended_at IS NULL\\n          AND accounts.moved_to_account_id IS NULL\\n        ORDER BY rank DESC\\n        LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
       "render_path": null,
       "location": {
         "type": "method",
-        "class": "Mastodon::Snowflake",
-        "method": "define_timestamp_id"
+        "class": "Account",
+        "method": "search_for"
       },
-      "user_input": "SecureRandom.hex(16)",
+      "user_input": "textsearch",
       "confidence": "Medium",
       "note": ""
     },
@@ -201,23 +221,53 @@
       "note": ""
     },
     {
-      "warning_type": "Redirect",
-      "warning_code": 18,
-      "fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33",
-      "check_name": "Redirect",
-      "message": "Possible unprotected redirect",
-      "file": "app/controllers/media_controller.rb",
-      "line": 20,
-      "link": "https://brakemanscanner.org/docs/warning_types/redirect/",
-      "code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))",
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "app/models/tag_filter.rb",
+      "line": 50,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")",
       "render_path": null,
       "location": {
         "type": "method",
-        "class": "MediaController",
-        "method": "show"
+        "class": "TagFilter",
+        "method": "trending_scope"
       },
-      "user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)",
-      "confidence": "High",
+      "user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")",
+      "confidence": "Medium",
+      "note": ""
+    },
+    {
+      "warning_type": "Cross-Site Scripting",
+      "warning_code": 4,
+      "fingerprint": "cd5cfd7f40037fbfa753e494d7129df16e358bfc43ef0da3febafbf4ee1ed3ac",
+      "check_name": "LinkToHref",
+      "message": "Potentially unsafe model attribute in `link_to` href",
+      "file": "app/views/admin/trends/links/_preview_card.html.haml",
+      "line": 7,
+      "link": "https://brakemanscanner.org/docs/warning_types/link_to_href",
+      "code": "link_to((Unresolved Model).new.title, (Unresolved Model).new.url)",
+      "render_path": [
+        {
+          "type": "template",
+          "name": "admin/trends/links/index",
+          "line": 37,
+          "file": "app/views/admin/trends/links/index.html.haml",
+          "rendered": {
+            "name": "admin/trends/links/_preview_card",
+            "file": "app/views/admin/trends/links/_preview_card.html.haml"
+          }
+        }
+      ],
+      "location": {
+        "type": "template",
+        "template": "admin/trends/links/_preview_card"
+      },
+      "user_input": "(Unresolved Model).new.url",
+      "confidence": "Weak",
       "note": ""
     },
     {
@@ -227,7 +277,7 @@
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/account.rb",
-      "line": 495,
+      "line": 500,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
       "code": "find_by_sql([\"          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n          WHERE #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
       "render_path": null,
@@ -261,6 +311,6 @@
       "note": ""
     }
   ],
-  "updated": "2021-05-11 20:22:27 +0900",
-  "brakeman_version": "5.0.1"
+  "updated": "2021-11-14 05:26:09 +0100",
+  "brakeman_version": "5.1.2"
 }
diff --git a/config/locales/en.yml b/config/locales/en.yml
index be15ad4b0..c98b82801 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -674,8 +674,8 @@ en:
         desc_html: Affects hashtags that have not been previously disallowed
         title: Allow hashtags to trend without prior review
       trends:
-        desc_html: Publicly display previously reviewed hashtags that are currently trending
-        title: Trending hashtags
+        desc_html: Publicly display previously reviewed content that is currently trending
+        title: Trends
     site_uploads:
       delete: Delete uploaded file
       destroyed_msg: Site upload successfully deleted!
@@ -702,21 +702,51 @@ en:
       sidekiq_process_check:
         message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
     tags:
-      accounts_today: Unique uses today
-      accounts_week: Unique uses this week
-      breakdown: Breakdown of today's usage by source
-      last_active: Recently used
-      most_popular: Most popular
-      most_recent: Recently created
-      name: Hashtag
       review: Review status
-      reviewed: Reviewed
-      title: Hashtags
-      trending_right_now: Trending right now
-      unique_uses_today: "%{count} posting today"
-      unreviewed: Not reviewed
       updated_msg: Hashtag settings updated successfully
     title: Administration
+    trends:
+      allow: Allow
+      approved: Approved
+      disallow: Disallow
+      links:
+        allow: Allow link
+        allow_provider: Allow publisher
+        disallow: Disallow link
+        disallow_provider: Disallow publisher
+        shared_by_over_week:
+          one: Shared by one person over the last week
+          other: Shared by %{count} people over the last week
+        title: Trending links
+        usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday
+      pending_review: Pending review
+      preview_card_providers:
+        allowed: Links from this publisher can trend
+        rejected: Links from this publisher won't trend
+        title: Publishers
+      rejected: Rejected
+      tags:
+        current_score: Current score %{score}
+        dashboard:
+          tag_accounts_measure: unique uses
+          tag_languages_dimension: Top languages
+          tag_servers_dimension: Top servers
+          tag_servers_measure: different servers
+          tag_uses_measure: total uses
+        listable: Can be suggested
+        not_listable: Won't be suggested
+        not_trendable: Won't appear under trends
+        not_usable: Cannot be used
+        peaked_on_and_decaying: Peaked on %{date}, now decaying
+        title: Trending hashtags
+        trendable: Can appear under trends
+        trending_rank: 'Trending #%{rank}'
+        usable: Can be used
+        usage_comparison: Used %{today} times today, compared to %{yesterday} yesterday
+        used_by_over_week:
+          one: Used by one person over the last week
+          other: Used by %{count} people over the last week
+      title: Trends
     warning_presets:
       add_new: Add new
       delete: Delete
@@ -731,9 +761,16 @@ en:
       body: "%{reporter} has reported %{target}"
       body_remote: Someone from %{domain} has reported %{target}
       subject: New report for %{instance} (#%{id})
-    new_trending_tag:
-      body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
-      subject: New hashtag up for review on %{instance} (#%{name})
+    new_trending_links:
+      body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated.
+      no_approved_links: There are currently no approved trending links.
+      requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.
+      subject: New trending links up for review on %{instance}
+    new_trending_tags:
+      body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:'
+      no_approved_tags: There are currently no approved trending hashtags.
+      requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
+      subject: New trending hashtags up for review on %{instance}
   aliases:
     add_new: Create alias
     created_msg: Successfully created a new alias. You can now initiate the move from the old account.
@@ -940,7 +977,7 @@ en:
     changes_saved_msg: Changes successfully saved!
     copy: Copy
     delete: Delete
-    no_batch_actions_available: No batch actions available on this page
+    none: None
     order_by: Order by
     save_changes: Save changes
     validation_errors:
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index bf864748c..d6376782d 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -204,8 +204,8 @@ en:
         mention: Someone mentioned you
         pending_account: New account needs review
         reblog: Someone boosted your post
-        report: New report is submitted
-        trending_tag: An unreviewed hashtag is trending
+        report: A new report is submitted
+        trending_tag: A new trend requires approval
       rule:
         text: Rule
       tag:
diff --git a/config/navigation.rb b/config/navigation.rb
index 37bfd7549..477d1c9ff 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -34,12 +34,16 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? }
     n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? }
 
+    n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s|
+      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
+      s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
+    end
+
     n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s|
       s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
-      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
       s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}
       s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
diff --git a/config/routes.rb b/config/routes.rb
index 86f699516..c7317d173 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -301,12 +301,27 @@ Rails.application.routes.draw do
 
     resources :account_moderation_notes, only: [:create, :destroy]
     resource :follow_recommendations, only: [:show, :update]
+    resources :tags, only: [:show, :update]
 
-    resources :tags, only: [:index, :show, :update] do
-      collection do
-        post :approve_all
-        post :reject_all
-        post :batch
+    namespace :trends do
+      resources :links, only: [:index] do
+        collection do
+          post :batch
+        end
+      end
+
+      resources :tags, only: [:index] do
+        collection do
+          post :batch
+        end
+      end
+
+      namespace :links do
+        resources :preview_card_providers, only: [:index], path: :publishers do
+          collection do
+            post :batch
+          end
+        end
       end
     end
   end
@@ -399,7 +414,7 @@ Rails.application.routes.draw do
       resources :favourites,   only: [:index]
       resources :bookmarks,    only: [:index]
       resources :reports,      only: [:create]
-      resources :trends,       only: [:index]
+      resources :trends,       only: [:index], controller: 'trends/tags'
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
       resources :markers,      only: [:index, :create]
@@ -410,6 +425,11 @@ Rails.application.routes.draw do
 
       resources :apps, only: [:create]
 
+      namespace :trends do
+        resources :links, only: [:index]
+        resources :tags, only: [:index]
+      end
+
       namespace :emails do
         resources :confirmations, only: [:create]
       end
@@ -512,7 +532,9 @@ Rails.application.routes.draw do
           end
         end
 
-        resources :trends, only: [:index]
+        namespace :trends do
+          resources :tags, only: [:index]
+        end
 
         post :measures, to: 'measures#create'
         post :dimensions, to: 'dimensions#create'
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index eab74338e..9dde5a053 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -13,9 +13,13 @@
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
     queue: scheduler
-  trending_tags_scheduler:
+  trends_refresh_scheduler:
     every: '5m'
-    class: Scheduler::TrendingTagsScheduler
+    class: Scheduler::Trends::RefreshScheduler
+    queue: scheduler
+  trends_review_notifications_scheduler:
+    every: '2h'
+    class: Scheduler::Trends::ReviewNotificationsScheduler
     queue: scheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
diff --git a/db/migrate/20211031031021_create_preview_card_providers.rb b/db/migrate/20211031031021_create_preview_card_providers.rb
new file mode 100644
index 000000000..0bd46198e
--- /dev/null
+++ b/db/migrate/20211031031021_create_preview_card_providers.rb
@@ -0,0 +1,12 @@
+class CreatePreviewCardProviders < ActiveRecord::Migration[6.1]
+  def change
+    create_table :preview_card_providers do |t|
+      t.string :domain, null: false, default: '', index: { unique: true }
+      t.attachment :icon
+      t.boolean :trendable
+      t.datetime :reviewed_at
+      t.datetime :requested_review_at
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20211112011713_add_language_to_preview_cards.rb b/db/migrate/20211112011713_add_language_to_preview_cards.rb
new file mode 100644
index 000000000..995934de4
--- /dev/null
+++ b/db/migrate/20211112011713_add_language_to_preview_cards.rb
@@ -0,0 +1,7 @@
+class AddLanguageToPreviewCards < ActiveRecord::Migration[6.1]
+  def change
+    add_column :preview_cards, :language, :string
+    add_column :preview_cards, :max_score, :float
+    add_column :preview_cards, :max_score_at, :datetime
+  end
+end
diff --git a/db/migrate/20211115032527_add_trendable_to_preview_cards.rb b/db/migrate/20211115032527_add_trendable_to_preview_cards.rb
new file mode 100644
index 000000000..87bf3d7a2
--- /dev/null
+++ b/db/migrate/20211115032527_add_trendable_to_preview_cards.rb
@@ -0,0 +1,5 @@
+class AddTrendableToPreviewCards < ActiveRecord::Migration[6.1]
+  def change
+    add_column :preview_cards, :trendable, :boolean
+  end
+end
diff --git a/db/migrate/20211123212714_add_link_type_to_preview_cards.rb b/db/migrate/20211123212714_add_link_type_to_preview_cards.rb
new file mode 100644
index 000000000..9f57e0219
--- /dev/null
+++ b/db/migrate/20211123212714_add_link_type_to_preview_cards.rb
@@ -0,0 +1,5 @@
+class AddLinkTypeToPreviewCards < ActiveRecord::Migration[6.1]
+  def change
+    add_column :preview_cards, :link_type, :int
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2376afff7..00969daf1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_08_08_071221) do
+ActiveRecord::Schema.define(version: 2021_11_23_212714) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -689,6 +689,20 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
     t.index ["status_id"], name: "index_polls_on_status_id"
   end
 
+  create_table "preview_card_providers", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.string "icon_file_name"
+    t.string "icon_content_type"
+    t.bigint "icon_file_size"
+    t.datetime "icon_updated_at"
+    t.boolean "trendable"
+    t.datetime "reviewed_at"
+    t.datetime "requested_review_at"
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["domain"], name: "index_preview_card_providers_on_domain", unique: true
+  end
+
   create_table "preview_cards", force: :cascade do |t|
     t.string "url", default: "", null: false
     t.string "title", default: "", null: false
@@ -710,6 +724,11 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
     t.string "embed_url", default: "", null: false
     t.integer "image_storage_schema_version"
     t.string "blurhash"
+    t.string "language"
+    t.float "max_score"
+    t.datetime "max_score_at"
+    t.boolean "trendable"
+    t.integer "link_type"
     t.index ["url"], name: "index_preview_cards_on_url", unique: true
   end
 
diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb
index 8e2d82a97..fe0dc1722 100644
--- a/lib/mastodon/snowflake.rb
+++ b/lib/mastodon/snowflake.rb
@@ -84,10 +84,7 @@ module Mastodon::Snowflake
               -- Take the first two bytes (four hex characters)
               substr(
                 -- Of the MD5 hash of the data we documented
-                md5(table_name ||
-                  '#{SecureRandom.hex(16)}' ||
-                  time_part::text
-                ),
+                md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),
                 1, 4
               )
             -- And turn it into a bigint
diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake
index d004c5751..bbf7f20ee 100644
--- a/lib/tasks/repo.rake
+++ b/lib/tasks/repo.rake
@@ -96,7 +96,7 @@ namespace :repo do
     end.uniq.compact
 
     missing_available_locales = locales_in_files - I18n.available_locales
-    missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) }
+    missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) }
 
     critical = false
 
diff --git a/spec/controllers/admin/tags_controller_spec.rb b/spec/controllers/admin/tags_controller_spec.rb
index 9145d887d..85c801a9c 100644
--- a/spec/controllers/admin/tags_controller_spec.rb
+++ b/spec/controllers/admin/tags_controller_spec.rb
@@ -9,18 +9,6 @@ RSpec.describe Admin::TagsController, type: :controller do
     sign_in Fabricate(:user, admin: true)
   end
 
-  describe 'GET #index' do
-    let!(:tag) { Fabricate(:tag) }
-
-    before do
-      get :index
-    end
-
-    it 'returns status 200' do
-      expect(response).to have_http_status(200)
-    end
-  end
-
   describe 'GET #show' do
     let!(:tag) { Fabricate(:tag) }
 
diff --git a/spec/controllers/api/v1/trends/tags_controller_spec.rb b/spec/controllers/api/v1/trends/tags_controller_spec.rb
new file mode 100644
index 000000000..e2e26dcab
--- /dev/null
+++ b/spec/controllers/api/v1/trends/tags_controller_spec.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Api::V1::Trends::TagsController, type: :controller do
+  render_views
+
+  describe 'GET #index' do
+    before do
+      trending_tags = double()
+
+      allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag))
+      allow(Trends).to receive(:tags).and_return(trending_tags)
+
+      get :index
+    end
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/trends_controller_spec.rb b/spec/controllers/api/v1/trends_controller_spec.rb
deleted file mode 100644
index 91e0d18fe..000000000
--- a/spec/controllers/api/v1/trends_controller_spec.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Api::V1::TrendsController, type: :controller do
-  render_views
-
-  describe 'GET #index' do
-    before do
-      allow(TrendingTags).to receive(:get).and_return(Fabricate.times(10, :tag))
-      get :index
-    end
-
-    it 'returns http success' do
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/helpers/languages_helper_spec.rb b/spec/helpers/languages_helper_spec.rb
new file mode 100644
index 000000000..6db617824
--- /dev/null
+++ b/spec/helpers/languages_helper_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe LanguagesHelper do
+  describe 'the HUMAN_LOCALES constant' do
+    it 'includes all I18n locales' do
+      expect(described_class::HUMAN_LOCALES.keys).to include(*I18n.available_locales)
+    end
+  end
+
+  describe 'human_locale' do
+    it 'finds the human readable local description from a key' do
+      expect(helper.human_locale(:en)).to eq('English')
+    end
+  end
+end
diff --git a/spec/helpers/settings_helper_spec.rb b/spec/helpers/settings_helper_spec.rb
deleted file mode 100644
index 092c37583..000000000
--- a/spec/helpers/settings_helper_spec.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe SettingsHelper do
-  describe 'the HUMAN_LOCALES constant' do
-    it 'includes all I18n locales' do
-      options = I18n.available_locales
-
-      expect(described_class::HUMAN_LOCALES.keys).to include(*options)
-    end
-  end
-
-  describe 'human_locale' do
-    it 'finds the human readable local description from a key' do
-      # Ensure the value is as we expect
-      expect(described_class::HUMAN_LOCALES[:en]).to eq('English')
-
-      expect(helper.human_locale(:en)).to eq('English')
-    end
-  end
-end
diff --git a/spec/mailers/previews/admin_mailer_preview.rb b/spec/mailers/previews/admin_mailer_preview.rb
index 561a56b78..75ffbbf40 100644
--- a/spec/mailers/previews/admin_mailer_preview.rb
+++ b/spec/mailers/previews/admin_mailer_preview.rb
@@ -5,4 +5,14 @@ class AdminMailerPreview < ActionMailer::Preview
   def new_pending_account
     AdminMailer.new_pending_account(Account.first, User.pending.first)
   end
+
+  # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags
+  def new_trending_tags
+    AdminMailer.new_trending_tags(Account.first, Tag.limit(3))
+  end
+
+  # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links
+  def new_trending_links
+    AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3))
+  end
 end
diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb
deleted file mode 100644
index dfbc7d6f8..000000000
--- a/spec/models/trending_tags_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe TrendingTags do
-  describe '.record_use!' do
-    pending
-  end
-
-  describe '.update!' do
-    let!(:at_time) { Time.now.utc }
-    let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
-    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
-    let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
-
-    before do
-      allow(Redis.current).to receive(:pfcount) do |key|
-        case key
-        when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
-          2
-        when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
-          16
-        when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
-          0
-        when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
-          4
-        when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
-          13
-        end
-      end
-
-      Redis.current.zadd('trending_tags', 0.9, tag3.id)
-      Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
-
-      tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
-
-      described_class.update!(at_time)
-    end
-
-    it 'calculates and re-calculates scores' do
-      expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
-    end
-
-    it 'omits hashtags below threshold' do
-      expect(described_class.get(10, filtered: false)).to_not include(tag2)
-    end
-
-    it 'decays scores' do
-      expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
-    end
-  end
-
-  describe '.trending?' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
-    end
-
-    it 'returns true if the hashtag is within limit' do
-      Redis.current.zadd('trending_tags', 11, tag.id)
-      expect(described_class.trending?(tag)).to be true
-    end
-
-    it 'returns false if the hashtag is outside the limit' do
-      Redis.current.zadd('trending_tags', 0, tag.id)
-      expect(described_class.trending?(tag)).to be false
-    end
-  end
-end
diff --git a/spec/models/trends/tags_spec.rb b/spec/models/trends/tags_spec.rb
new file mode 100644
index 000000000..4f98c6aa4
--- /dev/null
+++ b/spec/models/trends/tags_spec.rb
@@ -0,0 +1,67 @@
+require 'rails_helper'
+
+RSpec.describe Trends::Tags do
+  subject { described_class.new(threshold: 5, review_threshold: 10) }
+
+  let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) }
+
+  describe '#add' do
+    let(:tag) { Fabricate(:tag) }
+
+    before do
+      subject.add(tag, 1, at_time)
+    end
+
+    it 'records history' do
+      expect(tag.history.get(at_time).accounts).to eq 1
+    end
+
+    it 'records use' do
+      expect(subject.send(:recently_used_ids, at_time)).to eq [tag.id]
+    end
+  end
+
+  describe '#get' do
+    pending
+  end
+
+  describe '#refresh' do
+    let!(:today) { at_time }
+    let!(:yesterday) { today - 1.day }
+
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
+
+    before do
+      2.times  { |i| subject.add(tag1, i, yesterday) }
+      13.times { |i| subject.add(tag3, i, yesterday) }
+      16.times { |i| subject.add(tag1, i, today) }
+      4.times  { |i| subject.add(tag2, i, today) }
+    end
+
+    context do
+      before do
+        subject.refresh(yesterday + 12.hours)
+        subject.refresh(at_time)
+      end
+
+      it 'calculates and re-calculates scores' do
+        expect(subject.get(false, 10)).to eq [tag1, tag3]
+      end
+
+      it 'omits hashtags below threshold' do
+        expect(subject.get(false, 10)).to_not include(tag2)
+      end
+    end
+
+    it 'decays scores' do
+      subject.refresh(yesterday + 12.hours)
+      original_score = subject.score(tag3.id)
+      expect(original_score).to eq 144.0
+      subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife])
+      decayed_score = subject.score(tag3.id)
+      expect(decayed_score).to be <= original_score / 2
+    end
+  end
+end
-- 
cgit 


From 013bee6afb9f76c354e5f8d096d9c85d1ac484ce Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 25 Nov 2021 23:46:30 +0100
Subject: Fix filtering DMs from non-followed users (#17042)

---
 app/services/notify_service.rb       | 43 +++++++++++++++++++++++++++++++++++-
 spec/services/notify_service_spec.rb | 17 ++++++++++++--
 2 files changed, 57 insertions(+), 3 deletions(-)

(limited to 'app/services')

diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index a1b5ca1e3..09e28b76b 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -67,8 +67,49 @@ class NotifyService < BaseService
     message? && @notification.target_status.direct_visibility?
   end
 
+  # Returns true if the sender has been mentionned by the recipient up the thread
   def response_to_recipient?
-    @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility?
+    return false if @notification.target_status.in_reply_to_id.nil?
+
+    # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads
+    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero?
+      WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender) AS (
+          SELECT
+            s.id, s.in_reply_to_id, (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END)
+          FROM statuses s
+          WHERE s.id = :id
+        UNION ALL
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END)
+          FROM ancestors st
+          JOIN statuses s ON s.id = st.in_reply_to_id
+          WHERE st.replying_to_sender IS FALSE
+      )
+      SELECT COUNT(*)
+      FROM ancestors st
+      JOIN statuses s ON s.id = st.id
+      WHERE st.replying_to_sender IS TRUE AND s.visibility = 3
+    SQL
   end
 
   def from_staff?
diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb
index f2cb22c5e..7433866b7 100644
--- a/spec/services/notify_service_spec.rb
+++ b/spec/services/notify_service_spec.rb
@@ -64,8 +64,9 @@ RSpec.describe NotifyService, type: :service do
         is_expected.to_not change(Notification, :count)
       end
 
-      context 'if the message chain initiated by recipient, but is not direct message' do
+      context 'if the message chain is initiated by recipient, but is not direct message' do
         let(:reply_to) { Fabricate(:status, account: recipient) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
         it 'does not notify' do
@@ -73,8 +74,20 @@ RSpec.describe NotifyService, type: :service do
         end
       end
 
-      context 'if the message chain initiated by recipient and is direct message' do
+      context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do
+        let(:reply_to) { Fabricate(:status, account: recipient) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
+        let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) }
+        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) }
+
+        it 'does not notify' do
+          is_expected.to_not change(Notification, :count)
+        end
+      end
+
+      context 'if the message chain is initiated by the recipient with a mention to the sender' do
         let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
         it 'does notify' do
-- 
cgit 


From 7de0ee7aba86cffeaeffded7e0699214fb64364e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 26 Nov 2021 05:58:18 +0100
Subject: Remove Keybase integration (#17045)

---
 app/controllers/api/proofs_controller.rb           |  23 ---
 .../api/v1/accounts/identity_proofs_controller.rb  |   3 +-
 .../settings/identity_proofs_controller.rb         |  60 -------
 .../well_known/keybase_proof_config_controller.rb  |   9 -
 app/javascript/mastodon/actions/identity_proofs.js |  31 ----
 .../mastodon/features/account/components/header.js |  16 +-
 .../features/account_timeline/components/header.js |   4 +-
 .../containers/header_container.js                 |   2 -
 .../mastodon/features/account_timeline/index.js    |   2 -
 .../mastodon/reducers/identity_proofs.js           |  25 ---
 app/javascript/mastodon/reducers/index.js          |   2 -
 app/javascript/styles/mastodon/forms.scss          |  62 -------
 app/lib/activitypub/adapter.rb                     |   1 -
 app/lib/proof_provider.rb                          |  12 --
 app/lib/proof_provider/keybase.rb                  |  69 --------
 app/lib/proof_provider/keybase/badge.rb            |  45 -----
 .../proof_provider/keybase/config_serializer.rb    |  76 ---------
 app/lib/proof_provider/keybase/serializer.rb       |  25 ---
 app/lib/proof_provider/keybase/verifier.rb         |  59 -------
 app/lib/proof_provider/keybase/worker.rb           |  32 ----
 app/models/account_identity_proof.rb               |  46 -----
 app/models/concerns/account_associations.rb        |   3 +-
 app/models/concerns/account_merging.rb             |   2 +-
 app/serializers/activitypub/actor_serializer.rb    |   5 +-
 app/serializers/rest/identity_proof_serializer.rb  |  17 --
 .../activitypub/process_account_service.rb         |  26 ---
 app/services/delete_account_service.rb             |   2 -
 app/views/accounts/_bio.html.haml                  |  10 +-
 app/views/admin/accounts/show.html.haml            |  12 +-
 .../settings/identity_proofs/_proof.html.haml      |  21 ---
 app/views/settings/identity_proofs/index.html.haml |  17 --
 app/views/settings/identity_proofs/new.html.haml   |  36 ----
 config/locales/en.yml                              |  21 ---
 config/navigation.rb                               |   1 -
 config/routes.rb                                   |   6 -
 .../20211126000907_drop_account_identity_proofs.rb |  13 ++
 db/schema.rb                                       |  15 +-
 spec/controllers/api/proofs_controller_spec.rb     |  93 -----------
 .../settings/identity_proofs_controller_spec.rb    | 186 ---------------------
 .../keybase_proof_config_controller_spec.rb        |  15 --
 .../account_identity_proof_fabricator.rb           |   8 -
 spec/lib/proof_provider/keybase/verifier_spec.rb   |  82 ---------
 .../activitypub/process_account_service_spec.rb    |  45 -----
 43 files changed, 25 insertions(+), 1215 deletions(-)
 delete mode 100644 app/controllers/api/proofs_controller.rb
 delete mode 100644 app/controllers/settings/identity_proofs_controller.rb
 delete mode 100644 app/controllers/well_known/keybase_proof_config_controller.rb
 delete mode 100644 app/javascript/mastodon/actions/identity_proofs.js
 delete mode 100644 app/javascript/mastodon/reducers/identity_proofs.js
 delete mode 100644 app/lib/proof_provider.rb
 delete mode 100644 app/lib/proof_provider/keybase.rb
 delete mode 100644 app/lib/proof_provider/keybase/badge.rb
 delete mode 100644 app/lib/proof_provider/keybase/config_serializer.rb
 delete mode 100644 app/lib/proof_provider/keybase/serializer.rb
 delete mode 100644 app/lib/proof_provider/keybase/verifier.rb
 delete mode 100644 app/lib/proof_provider/keybase/worker.rb
 delete mode 100644 app/models/account_identity_proof.rb
 delete mode 100644 app/serializers/rest/identity_proof_serializer.rb
 delete mode 100644 app/views/settings/identity_proofs/_proof.html.haml
 delete mode 100644 app/views/settings/identity_proofs/index.html.haml
 delete mode 100644 app/views/settings/identity_proofs/new.html.haml
 create mode 100644 db/post_migrate/20211126000907_drop_account_identity_proofs.rb
 delete mode 100644 spec/controllers/api/proofs_controller_spec.rb
 delete mode 100644 spec/controllers/settings/identity_proofs_controller_spec.rb
 delete mode 100644 spec/controllers/well_known/keybase_proof_config_controller_spec.rb
 delete mode 100644 spec/fabricators/account_identity_proof_fabricator.rb
 delete mode 100644 spec/lib/proof_provider/keybase/verifier_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/api/proofs_controller.rb b/app/controllers/api/proofs_controller.rb
deleted file mode 100644
index dd32cd577..000000000
--- a/app/controllers/api/proofs_controller.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class Api::ProofsController < Api::BaseController
-  include AccountOwnedConcern
-
-  skip_before_action :require_authenticated_user!
-
-  before_action :set_provider
-
-  def index
-    render json: @account, serializer: @provider.serializer_class
-  end
-
-  private
-
-  def set_provider
-    @provider = ProofProvider.find(params[:provider]) || raise(ActiveRecord::RecordNotFound)
-  end
-
-  def username_param
-    params[:username]
-  end
-end
diff --git a/app/controllers/api/v1/accounts/identity_proofs_controller.rb b/app/controllers/api/v1/accounts/identity_proofs_controller.rb
index 4b5f6902c..48f293f47 100644
--- a/app/controllers/api/v1/accounts/identity_proofs_controller.rb
+++ b/app/controllers/api/v1/accounts/identity_proofs_controller.rb
@@ -5,8 +5,7 @@ class Api::V1::Accounts::IdentityProofsController < Api::BaseController
   before_action :set_account
 
   def index
-    @proofs = @account.suspended? ? [] : @account.identity_proofs.active
-    render json: @proofs, each_serializer: REST::IdentityProofSerializer
+    render json: []
   end
 
   private
diff --git a/app/controllers/settings/identity_proofs_controller.rb b/app/controllers/settings/identity_proofs_controller.rb
deleted file mode 100644
index bf2899da6..000000000
--- a/app/controllers/settings/identity_proofs_controller.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-class Settings::IdentityProofsController < Settings::BaseController
-  before_action :check_required_params, only: :new
-
-  def index
-    @proofs = AccountIdentityProof.where(account: current_account).order(provider: :asc, provider_username: :asc)
-    @proofs.each(&:refresh!)
-  end
-
-  def new
-    @proof = current_account.identity_proofs.new(
-      token: params[:token],
-      provider: params[:provider],
-      provider_username: params[:provider_username]
-    )
-
-    if current_account.username.casecmp(params[:username]).zero?
-      render layout: 'auth'
-    else
-      redirect_to settings_identity_proofs_path, alert: I18n.t('identity_proofs.errors.wrong_user', proving: params[:username], current: current_account.username)
-    end
-  end
-
-  def create
-    @proof = current_account.identity_proofs.where(provider: resource_params[:provider], provider_username: resource_params[:provider_username]).first_or_initialize(resource_params)
-    @proof.token = resource_params[:token]
-
-    if @proof.save
-      PostStatusService.new.call(current_user.account, text: post_params[:status_text]) if publish_proof?
-      redirect_to @proof.on_success_path(params[:user_agent])
-    else
-      redirect_to settings_identity_proofs_path, alert: I18n.t('identity_proofs.errors.failed', provider: @proof.provider.capitalize)
-    end
-  end
-
-  def destroy
-    @proof = current_account.identity_proofs.find(params[:id])
-    @proof.destroy!
-    redirect_to settings_identity_proofs_path, success: I18n.t('identity_proofs.removed')
-  end
-
-  private
-
-  def check_required_params
-    redirect_to settings_identity_proofs_path unless [:provider, :provider_username, :username, :token].all? { |k| params[k].present? }
-  end
-
-  def resource_params
-    params.require(:account_identity_proof).permit(:provider, :provider_username, :token)
-  end
-
-  def publish_proof?
-    ActiveModel::Type::Boolean.new.cast(post_params[:post_status])
-  end
-
-  def post_params
-    params.require(:account_identity_proof).permit(:post_status, :status_text)
-  end
-end
diff --git a/app/controllers/well_known/keybase_proof_config_controller.rb b/app/controllers/well_known/keybase_proof_config_controller.rb
deleted file mode 100644
index e1d43ecbe..000000000
--- a/app/controllers/well_known/keybase_proof_config_controller.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module WellKnown
-  class KeybaseProofConfigController < ActionController::Base
-    def show
-      render json: {}, serializer: ProofProvider::Keybase::ConfigSerializer, root: 'keybase_config'
-    end
-  end
-end
diff --git a/app/javascript/mastodon/actions/identity_proofs.js b/app/javascript/mastodon/actions/identity_proofs.js
deleted file mode 100644
index 103983956..000000000
--- a/app/javascript/mastodon/actions/identity_proofs.js
+++ /dev/null
@@ -1,31 +0,0 @@
-import api from '../api';
-
-export const IDENTITY_PROOFS_ACCOUNT_FETCH_REQUEST = 'IDENTITY_PROOFS_ACCOUNT_FETCH_REQUEST';
-export const IDENTITY_PROOFS_ACCOUNT_FETCH_SUCCESS = 'IDENTITY_PROOFS_ACCOUNT_FETCH_SUCCESS';
-export const IDENTITY_PROOFS_ACCOUNT_FETCH_FAIL    = 'IDENTITY_PROOFS_ACCOUNT_FETCH_FAIL';
-
-export const fetchAccountIdentityProofs = accountId => (dispatch, getState) => {
-  dispatch(fetchAccountIdentityProofsRequest(accountId));
-
-  api(getState).get(`/api/v1/accounts/${accountId}/identity_proofs`)
-    .then(({ data }) => dispatch(fetchAccountIdentityProofsSuccess(accountId, data)))
-    .catch(err => dispatch(fetchAccountIdentityProofsFail(accountId, err)));
-};
-
-export const fetchAccountIdentityProofsRequest = id => ({
-  type: IDENTITY_PROOFS_ACCOUNT_FETCH_REQUEST,
-  id,
-});
-
-export const fetchAccountIdentityProofsSuccess = (accountId, identity_proofs) => ({
-  type: IDENTITY_PROOFS_ACCOUNT_FETCH_SUCCESS,
-  accountId,
-  identity_proofs,
-});
-
-export const fetchAccountIdentityProofsFail = (accountId, err) => ({
-  type: IDENTITY_PROOFS_ACCOUNT_FETCH_FAIL,
-  accountId,
-  err,
-  skipNotFound: true,
-});
diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js
index 4d0a828c7..48ec49d81 100644
--- a/app/javascript/mastodon/features/account/components/header.js
+++ b/app/javascript/mastodon/features/account/components/header.js
@@ -123,7 +123,7 @@ class Header extends ImmutablePureComponent {
   }
 
   render () {
-    const { account, intl, domain, identity_proofs } = this.props;
+    const { account, intl, domain } = this.props;
 
     if (!account) {
       return null;
@@ -297,20 +297,8 @@ class Header extends ImmutablePureComponent {
 
           <div className='account__header__extra'>
             <div className='account__header__bio'>
-              {(fields.size > 0 || identity_proofs.size > 0) && (
+              {fields.size > 0 && (
                 <div className='account__header__fields'>
-                  {identity_proofs.map((proof, i) => (
-                    <dl key={i}>
-                      <dt dangerouslySetInnerHTML={{ __html: proof.get('provider') }} />
-
-                      <dd className='verified'>
-                        <a href={proof.get('proof_url')} target='_blank' rel='noopener noreferrer'><span title={intl.formatMessage(messages.linkVerifiedOn, { date: intl.formatDate(proof.get('updated_at'), dateFormatOptions) })}>
-                          <Icon id='check' className='verified__mark' />
-                        </span></a>
-                        <a href={proof.get('profile_url')} target='_blank' rel='noopener noreferrer'><span dangerouslySetInnerHTML={{ __html: ' '+proof.get('provider_username') }} /></a>
-                      </dd>
-                    </dl>
-                  ))}
                   {fields.map((pair, i) => (
                     <dl key={i}>
                       <dt dangerouslySetInnerHTML={{ __html: pair.get('name_emojified') }} title={pair.get('name')} className='translate' />
diff --git a/app/javascript/mastodon/features/account_timeline/components/header.js b/app/javascript/mastodon/features/account_timeline/components/header.js
index 17b693600..33bea4c17 100644
--- a/app/javascript/mastodon/features/account_timeline/components/header.js
+++ b/app/javascript/mastodon/features/account_timeline/components/header.js
@@ -11,7 +11,6 @@ export default class Header extends ImmutablePureComponent {
 
   static propTypes = {
     account: ImmutablePropTypes.map,
-    identity_proofs: ImmutablePropTypes.list,
     onFollow: PropTypes.func.isRequired,
     onBlock: PropTypes.func.isRequired,
     onMention: PropTypes.func.isRequired,
@@ -92,7 +91,7 @@ export default class Header extends ImmutablePureComponent {
   }
 
   render () {
-    const { account, hideTabs, identity_proofs } = this.props;
+    const { account, hideTabs } = this.props;
 
     if (account === null) {
       return null;
@@ -104,7 +103,6 @@ export default class Header extends ImmutablePureComponent {
 
         <InnerHeader
           account={account}
-          identity_proofs={identity_proofs}
           onFollow={this.handleFollow}
           onBlock={this.handleBlock}
           onMention={this.handleMention}
diff --git a/app/javascript/mastodon/features/account_timeline/containers/header_container.js b/app/javascript/mastodon/features/account_timeline/containers/header_container.js
index e12019547..b3f8521cb 100644
--- a/app/javascript/mastodon/features/account_timeline/containers/header_container.js
+++ b/app/javascript/mastodon/features/account_timeline/containers/header_container.js
@@ -21,7 +21,6 @@ import { openModal } from '../../../actions/modal';
 import { blockDomain, unblockDomain } from '../../../actions/domain_blocks';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
 import { unfollowModal } from '../../../initial_state';
-import { List as ImmutableList } from 'immutable';
 
 const messages = defineMessages({
   unfollowConfirm: { id: 'confirmations.unfollow.confirm', defaultMessage: 'Unfollow' },
@@ -34,7 +33,6 @@ const makeMapStateToProps = () => {
   const mapStateToProps = (state, { accountId }) => ({
     account: getAccount(state, accountId),
     domain: state.getIn(['meta', 'domain']),
-    identity_proofs: state.getIn(['identity_proofs', accountId], ImmutableList()),
   });
 
   return mapStateToProps;
diff --git a/app/javascript/mastodon/features/account_timeline/index.js b/app/javascript/mastodon/features/account_timeline/index.js
index 20f1dba9f..37df2818b 100644
--- a/app/javascript/mastodon/features/account_timeline/index.js
+++ b/app/javascript/mastodon/features/account_timeline/index.js
@@ -12,7 +12,6 @@ import ColumnBackButton from '../../components/column_back_button';
 import { List as ImmutableList } from 'immutable';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { FormattedMessage } from 'react-intl';
-import { fetchAccountIdentityProofs } from '../../actions/identity_proofs';
 import MissingIndicator from 'mastodon/components/missing_indicator';
 import TimelineHint from 'mastodon/components/timeline_hint';
 import { me } from 'mastodon/initial_state';
@@ -80,7 +79,6 @@ class AccountTimeline extends ImmutablePureComponent {
     const { accountId, withReplies, dispatch } = this.props;
 
     dispatch(fetchAccount(accountId));
-    dispatch(fetchAccountIdentityProofs(accountId));
 
     if (!withReplies) {
       dispatch(expandAccountFeaturedTimeline(accountId));
diff --git a/app/javascript/mastodon/reducers/identity_proofs.js b/app/javascript/mastodon/reducers/identity_proofs.js
deleted file mode 100644
index 58af0a5fa..000000000
--- a/app/javascript/mastodon/reducers/identity_proofs.js
+++ /dev/null
@@ -1,25 +0,0 @@
-import { Map as ImmutableMap, fromJS } from 'immutable';
-import {
-  IDENTITY_PROOFS_ACCOUNT_FETCH_REQUEST,
-  IDENTITY_PROOFS_ACCOUNT_FETCH_SUCCESS,
-  IDENTITY_PROOFS_ACCOUNT_FETCH_FAIL,
-} from '../actions/identity_proofs';
-
-const initialState = ImmutableMap();
-
-export default function identityProofsReducer(state = initialState, action) {
-  switch(action.type) {
-  case IDENTITY_PROOFS_ACCOUNT_FETCH_REQUEST:
-    return state.set('isLoading', true);
-  case IDENTITY_PROOFS_ACCOUNT_FETCH_FAIL:
-    return state.set('isLoading', false);
-  case IDENTITY_PROOFS_ACCOUNT_FETCH_SUCCESS:
-    return state.update(identity_proofs => identity_proofs.withMutations(map => {
-      map.set('isLoading', false);
-      map.set('loaded', true);
-      map.set(action.accountId, fromJS(action.identity_proofs));
-    }));
-  default:
-    return state;
-  }
-};
diff --git a/app/javascript/mastodon/reducers/index.js b/app/javascript/mastodon/reducers/index.js
index e518c8228..53e2dd681 100644
--- a/app/javascript/mastodon/reducers/index.js
+++ b/app/javascript/mastodon/reducers/index.js
@@ -32,7 +32,6 @@ import filters from './filters';
 import conversations from './conversations';
 import suggestions from './suggestions';
 import polls from './polls';
-import identity_proofs from './identity_proofs';
 import trends from './trends';
 import missed_updates from './missed_updates';
 import announcements from './announcements';
@@ -69,7 +68,6 @@ const reducers = {
   notifications,
   height_cache,
   custom_emojis,
-  identity_proofs,
   lists,
   listEditor,
   listAdder,
diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss
index 5b71b6334..65f53471d 100644
--- a/app/javascript/styles/mastodon/forms.scss
+++ b/app/javascript/styles/mastodon/forms.scss
@@ -999,68 +999,6 @@ code {
   }
 }
 
-.connection-prompt {
-  margin-bottom: 25px;
-
-  .fa-link {
-    background-color: darken($ui-base-color, 4%);
-    border-radius: 100%;
-    font-size: 24px;
-    padding: 10px;
-  }
-
-  &__column {
-    align-items: center;
-    display: flex;
-    flex: 1;
-    flex-direction: column;
-    flex-shrink: 1;
-    max-width: 50%;
-
-    &-sep {
-      align-self: center;
-      flex-grow: 0;
-      overflow: visible;
-      position: relative;
-      z-index: 1;
-    }
-
-    p {
-      word-break: break-word;
-    }
-  }
-
-  .account__avatar {
-    margin-bottom: 20px;
-  }
-
-  &__connection {
-    background-color: lighten($ui-base-color, 8%);
-    box-shadow: 0 0 15px rgba($base-shadow-color, 0.2);
-    border-radius: 4px;
-    padding: 25px 10px;
-    position: relative;
-    text-align: center;
-
-    &::after {
-      background-color: darken($ui-base-color, 4%);
-      content: '';
-      display: block;
-      height: 100%;
-      left: 50%;
-      position: absolute;
-      top: 0;
-      width: 1px;
-    }
-  }
-
-  &__row {
-    align-items: flex-start;
-    display: flex;
-    flex-direction: row;
-  }
-}
-
 .input.user_confirm_password,
 .input.user_website {
   &:not(.field_with_errors) {
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index 2d6b87659..776e1d3da 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -18,7 +18,6 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
     atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
     conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
     focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
-    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
     blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
     discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
     voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
diff --git a/app/lib/proof_provider.rb b/app/lib/proof_provider.rb
deleted file mode 100644
index 102c50f4f..000000000
--- a/app/lib/proof_provider.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module ProofProvider
-  SUPPORTED_PROVIDERS = %w(keybase).freeze
-
-  def self.find(identifier, proof = nil)
-    case identifier
-    when 'keybase'
-      ProofProvider::Keybase.new(proof)
-    end
-  end
-end
diff --git a/app/lib/proof_provider/keybase.rb b/app/lib/proof_provider/keybase.rb
deleted file mode 100644
index 8e51d7146..000000000
--- a/app/lib/proof_provider/keybase.rb
+++ /dev/null
@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-class ProofProvider::Keybase
-  BASE_URL = ENV.fetch('KEYBASE_BASE_URL', 'https://keybase.io')
-  DOMAIN   = ENV.fetch('KEYBASE_DOMAIN', Rails.configuration.x.web_domain)
-
-  class Error < StandardError; end
-
-  class ExpectedProofLiveError < Error; end
-
-  class UnexpectedResponseError < Error; end
-
-  def initialize(proof = nil)
-    @proof = proof
-  end
-
-  def serializer_class
-    ProofProvider::Keybase::Serializer
-  end
-
-  def worker_class
-    ProofProvider::Keybase::Worker
-  end
-
-  def validate!
-    unless @proof.token&.size == 66
-      @proof.errors.add(:base, I18n.t('identity_proofs.errors.keybase.invalid_token'))
-      return
-    end
-
-    # Do not perform synchronous validation for remote accounts
-    return if @proof.provider_username.blank? || !@proof.account.local?
-
-    if verifier.valid?
-      @proof.verified = true
-      @proof.live     = false
-    else
-      @proof.errors.add(:base, I18n.t('identity_proofs.errors.keybase.verification_failed', kb_username: @proof.provider_username))
-    end
-  end
-
-  def refresh!
-    worker_class.new.perform(@proof)
-  rescue ProofProvider::Keybase::Error
-    nil
-  end
-
-  def on_success_path(user_agent = nil)
-    verifier.on_success_path(user_agent)
-  end
-
-  def badge
-    @badge ||= ProofProvider::Keybase::Badge.new(@proof.account.username, @proof.provider_username, @proof.token, domain)
-  end
-
-  def verifier
-    @verifier ||= ProofProvider::Keybase::Verifier.new(@proof.account.username, @proof.provider_username, @proof.token, domain)
-  end
-
-  private
-
-  def domain
-    if @proof.account.local?
-      DOMAIN
-    else
-      @proof.account.domain
-    end
-  end
-end
diff --git a/app/lib/proof_provider/keybase/badge.rb b/app/lib/proof_provider/keybase/badge.rb
deleted file mode 100644
index f587b1cc7..000000000
--- a/app/lib/proof_provider/keybase/badge.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-class ProofProvider::Keybase::Badge
-  include RoutingHelper
-
-  def initialize(local_username, provider_username, token, domain)
-    @local_username    = local_username
-    @provider_username = provider_username
-    @token             = token
-    @domain            = domain
-  end
-
-  def proof_url
-    "#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}/sigchain\##{@token}"
-  end
-
-  def profile_url
-    "#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}"
-  end
-
-  def icon_url
-    "#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}/proof_badge/#{@token}?username=#{@local_username}&domain=#{@domain}"
-  end
-
-  def avatar_url
-    Rails.cache.fetch("proof_providers/keybase/#{@provider_username}/avatar_url", expires_in: 5.minutes) { remote_avatar_url } || default_avatar_url
-  end
-
-  private
-
-  def remote_avatar_url
-    request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/user/pic_url.json", params: { username: @provider_username })
-
-    request.perform do |res|
-      json = Oj.load(res.body_with_limit, mode: :strict)
-      json['pic_url'] if json.is_a?(Hash)
-    end
-  rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError
-    nil
-  end
-
-  def default_avatar_url
-    asset_pack_path('media/images/proof_providers/keybase.png')
-  end
-end
diff --git a/app/lib/proof_provider/keybase/config_serializer.rb b/app/lib/proof_provider/keybase/config_serializer.rb
deleted file mode 100644
index c6c364d31..000000000
--- a/app/lib/proof_provider/keybase/config_serializer.rb
+++ /dev/null
@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-class ProofProvider::Keybase::ConfigSerializer < ActiveModel::Serializer
-  include RoutingHelper
-  include ActionView::Helpers::TextHelper
-
-  attributes :version, :domain, :display_name, :username,
-             :brand_color, :logo, :description, :prefill_url,
-             :profile_url, :check_url, :check_path, :avatar_path,
-             :contact
-
-  def version
-    1
-  end
-
-  def domain
-    ProofProvider::Keybase::DOMAIN
-  end
-
-  def display_name
-    Setting.site_title
-  end
-
-  def logo
-    {
-      svg_black: full_asset_url(asset_pack_path('media/images/logo_transparent_black.svg')),
-      svg_white: full_asset_url(asset_pack_path('media/images/logo_transparent_white.svg')),
-      svg_full: full_asset_url(asset_pack_path('media/images/logo.svg')),
-      svg_full_darkmode: full_asset_url(asset_pack_path('media/images/logo.svg')),
-    }
-  end
-
-  def brand_color
-    '#282c37'
-  end
-
-  def description
-    strip_tags(Setting.site_short_description.presence || I18n.t('about.about_mastodon_html'))
-  end
-
-  def username
-    { min: 1, max: 30, re: '[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?' }
-  end
-
-  def prefill_url
-    params = {
-      provider: 'keybase',
-      token: '%{sig_hash}',
-      provider_username: '%{kb_username}',
-      username: '%{username}',
-      user_agent: '%{kb_ua}',
-    }
-
-    CGI.unescape(new_settings_identity_proof_url(params))
-  end
-
-  def profile_url
-    CGI.unescape(short_account_url('%{username}'))
-  end
-
-  def check_url
-    CGI.unescape(api_proofs_url(username: '%{username}', provider: 'keybase'))
-  end
-
-  def check_path
-    ['signatures']
-  end
-
-  def avatar_path
-    ['avatar']
-  end
-
-  def contact
-    [Setting.site_contact_email.presence || 'unknown'].compact
-  end
-end
diff --git a/app/lib/proof_provider/keybase/serializer.rb b/app/lib/proof_provider/keybase/serializer.rb
deleted file mode 100644
index d29283600..000000000
--- a/app/lib/proof_provider/keybase/serializer.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-class ProofProvider::Keybase::Serializer < ActiveModel::Serializer
-  include RoutingHelper
-
-  attribute :avatar
-
-  has_many :identity_proofs, key: :signatures
-
-  def avatar
-    full_asset_url(object.avatar_original_url)
-  end
-
-  class AccountIdentityProofSerializer < ActiveModel::Serializer
-    attributes :sig_hash, :kb_username
-
-    def sig_hash
-      object.token
-    end
-
-    def kb_username
-      object.provider_username
-    end
-  end
-end
diff --git a/app/lib/proof_provider/keybase/verifier.rb b/app/lib/proof_provider/keybase/verifier.rb
deleted file mode 100644
index af69b1bfc..000000000
--- a/app/lib/proof_provider/keybase/verifier.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-class ProofProvider::Keybase::Verifier
-  def initialize(local_username, provider_username, token, domain)
-    @local_username    = local_username
-    @provider_username = provider_username
-    @token             = token
-    @domain            = domain
-  end
-
-  def valid?
-    request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/sig/proof_valid.json", params: query_params)
-
-    request.perform do |res|
-      json = Oj.load(res.body_with_limit, mode: :strict)
-
-      if json.is_a?(Hash)
-        json.fetch('proof_valid', false)
-      else
-        false
-      end
-    end
-  rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError
-    false
-  end
-
-  def on_success_path(user_agent = nil)
-    url = Addressable::URI.parse("#{ProofProvider::Keybase::BASE_URL}/_/proof_creation_success")
-    url.query_values = query_params.merge(kb_ua: user_agent || 'unknown')
-    url.to_s
-  end
-
-  def status
-    request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/sig/proof_live.json", params: query_params)
-
-    request.perform do |res|
-      raise ProofProvider::Keybase::UnexpectedResponseError unless res.code == 200
-
-      json = Oj.load(res.body_with_limit, mode: :strict)
-
-      raise ProofProvider::Keybase::UnexpectedResponseError unless json.is_a?(Hash) && json.key?('proof_valid') && json.key?('proof_live')
-
-      json
-    end
-  rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError
-    raise ProofProvider::Keybase::UnexpectedResponseError
-  end
-
-  private
-
-  def query_params
-    {
-      domain: @domain,
-      kb_username: @provider_username,
-      username: @local_username,
-      sig_hash: @token,
-    }
-  end
-end
diff --git a/app/lib/proof_provider/keybase/worker.rb b/app/lib/proof_provider/keybase/worker.rb
deleted file mode 100644
index bcdd18cc5..000000000
--- a/app/lib/proof_provider/keybase/worker.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-class ProofProvider::Keybase::Worker
-  include Sidekiq::Worker
-
-  sidekiq_options queue: 'pull', retry: 20, unique: :until_executed
-
-  sidekiq_retry_in do |count, exception|
-    # Retry aggressively when the proof is valid but not live in Keybase.
-    # This is likely because Keybase just hasn't noticed the proof being
-    # served from here yet.
-
-    if exception.class == ProofProvider::Keybase::ExpectedProofLiveError
-      case count
-      when 0..2 then 0.seconds
-      when 2..6 then 1.second
-      end
-    end
-  end
-
-  def perform(proof_id)
-    proof  = proof_id.is_a?(AccountIdentityProof) ? proof_id : AccountIdentityProof.find(proof_id)
-    status = proof.provider_instance.verifier.status
-
-    # If Keybase thinks the proof is valid, and it exists here in Mastodon,
-    # then it should be live. Keybase just has to notice that it's here
-    # and then update its state. That might take a couple seconds.
-    raise ProofProvider::Keybase::ExpectedProofLiveError if status['proof_valid'] && !status['proof_live']
-
-    proof.update!(verified: status['proof_valid'], live: status['proof_live'])
-  end
-end
diff --git a/app/models/account_identity_proof.rb b/app/models/account_identity_proof.rb
deleted file mode 100644
index 10b66cccf..000000000
--- a/app/models/account_identity_proof.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: account_identity_proofs
-#
-#  id                :bigint(8)        not null, primary key
-#  account_id        :bigint(8)
-#  provider          :string           default(""), not null
-#  provider_username :string           default(""), not null
-#  token             :text             default(""), not null
-#  verified          :boolean          default(FALSE), not null
-#  live              :boolean          default(FALSE), not null
-#  created_at        :datetime         not null
-#  updated_at        :datetime         not null
-#
-
-class AccountIdentityProof < ApplicationRecord
-  belongs_to :account
-
-  validates :provider, inclusion: { in: ProofProvider::SUPPORTED_PROVIDERS }
-  validates :provider_username, format: { with: /\A[a-z0-9_]+\z/i }, length: { minimum: 2, maximum: 30 }
-  validates :provider_username, uniqueness: { scope: [:account_id, :provider] }
-  validates :token, format: { with: /\A[a-f0-9]+\z/ }, length: { maximum: 66 }
-
-  validate :validate_with_provider, if: :token_changed?
-
-  scope :active, -> { where(verified: true, live: true) }
-
-  after_commit :queue_worker, if: :saved_change_to_token?
-
-  delegate :refresh!, :on_success_path, :badge, to: :provider_instance
-
-  def provider_instance
-    @provider_instance ||= ProofProvider.find(provider, self)
-  end
-
-  private
-
-  def queue_worker
-    provider_instance.worker_class.perform_async(id)
-  end
-
-  def validate_with_provider
-    provider_instance.validate!
-  end
-end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index f2a4eae77..f9e7a3bea 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -7,8 +7,7 @@ module AccountAssociations
     # Local users
     has_one :user, inverse_of: :account, dependent: :destroy
 
-    # Identity proofs
-    has_many :identity_proofs, class_name: 'AccountIdentityProof', dependent: :destroy, inverse_of: :account
+    # E2EE
     has_many :devices, dependent: :destroy, inverse_of: :account
 
     # Timelines
diff --git a/app/models/concerns/account_merging.rb b/app/models/concerns/account_merging.rb
index 8d37c6e56..119773e6b 100644
--- a/app/models/concerns/account_merging.rb
+++ b/app/models/concerns/account_merging.rb
@@ -13,7 +13,7 @@ module AccountMerging
 
     owned_classes = [
       Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
-      Follow, FollowRequest, Block, Mute, AccountIdentityProof,
+      Follow, FollowRequest, Block, Mute,
       AccountModerationNote, AccountPin, AccountStat, ListAccount,
       PollVote, Mention, AccountDeletionRequest, AccountNote, FollowRecommendationSuppression
     ]
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index a7d948976..48707aa16 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -6,8 +6,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   context :security
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
-                     :moved_to, :property_value, :identity_proof,
-                     :discoverable, :olm, :suspended
+                     :moved_to, :property_value, :discoverable, :olm, :suspended
 
   attributes :id, :type, :following, :followers,
              :inbox, :outbox, :featured, :featured_tags,
@@ -143,7 +142,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def virtual_attachments
-    object.suspended? ? [] : (object.fields + object.identity_proofs.active)
+    object.suspended? ? [] : object.fields
   end
 
   def moved_to
diff --git a/app/serializers/rest/identity_proof_serializer.rb b/app/serializers/rest/identity_proof_serializer.rb
deleted file mode 100644
index 0e7415935..000000000
--- a/app/serializers/rest/identity_proof_serializer.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-class REST::IdentityProofSerializer < ActiveModel::Serializer
-  attributes :provider, :provider_username, :updated_at, :proof_url, :profile_url
-
-  def proof_url
-    object.badge.proof_url
-  end
-
-  def profile_url
-    object.badge.profile_url
-  end
-
-  def provider
-    object.provider.capitalize
-  end
-end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 4ab6912e5..ec5140720 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -27,7 +27,6 @@ class ActivityPub::ProcessAccountService < BaseService
         create_account if @account.nil?
         update_account
         process_tags
-        process_attachments
 
         process_duplicate_accounts! if @options[:verified_webfinger]
       else
@@ -301,23 +300,6 @@ class ActivityPub::ProcessAccountService < BaseService
     end
   end
 
-  def process_attachments
-    return if @json['attachment'].blank?
-
-    previous_proofs = @account.identity_proofs.to_a
-    current_proofs  = []
-
-    as_array(@json['attachment']).each do |attachment|
-      next unless equals_or_includes?(attachment['type'], 'IdentityProof')
-      current_proofs << process_identity_proof(attachment)
-    end
-
-    previous_proofs.each do |previous_proof|
-      next if current_proofs.any? { |current_proof| current_proof.id == previous_proof.id }
-      previous_proof.delete
-    end
-  end
-
   def process_emoji(tag)
     return if skip_download?
     return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
@@ -334,12 +316,4 @@ class ActivityPub::ProcessAccountService < BaseService
     emoji.image_remote_url = image_url
     emoji.save
   end
-
-  def process_identity_proof(attachment)
-    provider          = attachment['signatureAlgorithm']
-    provider_username = attachment['name']
-    token             = attachment['signatureValue']
-
-    @account.identity_proofs.where(provider: provider, provider_username: provider_username).find_or_create_by(provider: provider, provider_username: provider_username, token: token)
-  end
 end
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index ac571d7e2..0e3fedfe7 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -17,7 +17,6 @@ class DeleteAccountService < BaseService
     domain_blocks
     featured_tags
     follow_requests
-    identity_proofs
     list_accounts
     migrations
     mute_relationships
@@ -45,7 +44,6 @@ class DeleteAccountService < BaseService
     domain_blocks
     featured_tags
     follow_requests
-    identity_proofs
     list_accounts
     migrations
     mute_relationships
diff --git a/app/views/accounts/_bio.html.haml b/app/views/accounts/_bio.html.haml
index efc26d136..e8a49a1aa 100644
--- a/app/views/accounts/_bio.html.haml
+++ b/app/views/accounts/_bio.html.haml
@@ -1,16 +1,8 @@
-- proofs = account.identity_proofs.active
 - fields = account.fields
 
 .public-account-bio
-  - unless fields.empty? && proofs.empty?
+  - unless fields.empty?
     .account__header__fields
-      - proofs.each do |proof|
-        %dl
-          %dt= proof.provider.capitalize
-          %dd.verified
-            = link_to fa_icon('check'), proof.badge.proof_url, class: 'verified__mark', title: t('accounts.link_verified_on', date: l(proof.updated_at))
-            = link_to proof.provider_username, proof.badge.profile_url
-
       - fields.each do |field|
         %dl
           %dt.emojify{ title: field.name }= Formatter.instance.format_field(account, field.name, custom_emojify: true)
diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml
index 66eb49342..2b6e28e8d 100644
--- a/app/views/admin/accounts/show.html.haml
+++ b/app/views/admin/accounts/show.html.haml
@@ -8,20 +8,12 @@
 = render 'application/card', account: @account
 
 - account = @account
-- proofs = account.identity_proofs.active
 - fields = account.fields
-- unless fields.empty? && proofs.empty? && account.note.blank?
+- unless fields.empty? && account.note.blank?
   .admin-account-bio
-    - unless fields.empty? && proofs.empty?
+    - unless fields.empty?
       %div
         .account__header__fields
-          - proofs.each do |proof|
-            %dl
-              %dt= proof.provider.capitalize
-              %dd.verified
-                = link_to fa_icon('check'), proof.badge.proof_url, class: 'verified__mark', title: t('accounts.link_verified_on', date: l(proof.updated_at))
-                = link_to proof.provider_username, proof.badge.profile_url
-
           - fields.each do |field|
             %dl
               %dt.emojify{ title: field.name }= Formatter.instance.format_field(account, field.name, custom_emojify: true)
diff --git a/app/views/settings/identity_proofs/_proof.html.haml b/app/views/settings/identity_proofs/_proof.html.haml
deleted file mode 100644
index 14e8e91be..000000000
--- a/app/views/settings/identity_proofs/_proof.html.haml
+++ /dev/null
@@ -1,21 +0,0 @@
-%tr
-  %td
-    = link_to proof.badge.profile_url, class: 'name-tag' do
-      = image_tag proof.badge.avatar_url, width: 15, height: 15, alt: '', class: 'avatar'
-      %span.username
-        = proof.provider_username
-        %span= "(#{proof.provider.capitalize})"
-
-  %td
-    - if proof.live?
-      %span.positive-hint
-        = fa_icon 'check-circle fw'
-        = t('identity_proofs.active')
-    - else
-      %span.negative-hint
-        = fa_icon 'times-circle fw'
-        = t('identity_proofs.inactive')
-
-  %td
-    = table_link_to 'external-link', t('identity_proofs.view_proof'), proof.badge.proof_url if proof.badge.proof_url
-    = table_link_to 'trash', t('identity_proofs.remove'), settings_identity_proof_path(proof), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') }
diff --git a/app/views/settings/identity_proofs/index.html.haml b/app/views/settings/identity_proofs/index.html.haml
deleted file mode 100644
index d0ea03ecd..000000000
--- a/app/views/settings/identity_proofs/index.html.haml
+++ /dev/null
@@ -1,17 +0,0 @@
-- content_for :page_title do
-  = t('settings.identity_proofs')
-
-%p= t('identity_proofs.explanation_html')
-
-- unless @proofs.empty?
-  %hr.spacer/
-
-  .table-wrapper
-    %table.table
-      %thead
-        %tr
-          %th= t('identity_proofs.identity')
-          %th= t('identity_proofs.status')
-          %th
-      %tbody
-        = render partial: 'settings/identity_proofs/proof', collection: @proofs, as: :proof
diff --git a/app/views/settings/identity_proofs/new.html.haml b/app/views/settings/identity_proofs/new.html.haml
deleted file mode 100644
index 5e4e9895d..000000000
--- a/app/views/settings/identity_proofs/new.html.haml
+++ /dev/null
@@ -1,36 +0,0 @@
-- content_for :page_title do
-  = t('identity_proofs.authorize_connection_prompt')
-
-.form-container
-  .oauth-prompt
-    %h2= t('identity_proofs.authorize_connection_prompt')
-
-  = simple_form_for @proof, url: settings_identity_proofs_url, html: { method: :post } do |f|
-    = f.input :provider, as: :hidden
-    = f.input :provider_username, as: :hidden
-    = f.input :token, as: :hidden
-
-    = hidden_field_tag :user_agent, params[:user_agent]
-
-    .connection-prompt
-      .connection-prompt__row.connection-prompt__connection
-        .connection-prompt__column
-          = image_tag current_account.avatar.url(:original), size: 96, class: 'account__avatar'
-
-          %p= t('identity_proofs.i_am_html', username: content_tag(:strong,current_account.username), service: site_hostname)
-
-        .connection-prompt__column.connection-prompt__column-sep
-          = fa_icon 'link'
-
-        .connection-prompt__column
-          = image_tag @proof.badge.avatar_url, size: 96, class: 'account__avatar'
-
-          %p= t('identity_proofs.i_am_html', username: content_tag(:strong, @proof.provider_username), service: @proof.provider.capitalize)
-
-    .connection-prompt__post
-      = f.input :post_status, label: t('identity_proofs.publicize_checkbox'), as: :boolean, wrapper: :with_label, :input_html => { checked: true }
-
-      = f.input :status_text, as: :text, input_html: { value: t('identity_proofs.publicize_toot', username: @proof.provider_username, service: @proof.provider.capitalize, url: @proof.badge.proof_url), rows: 4 }
-
-    = f.button :button, t('identity_proofs.authorize'), type: :submit
-    = link_to t('simple_form.no'), settings_identity_proofs_url, class: 'button negative'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index c98b82801..1aa96ba0f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -985,26 +985,6 @@ en:
       other: Something isn't quite right yet! Please review %{count} errors below
   html_validator:
     invalid_markup: 'contains invalid HTML markup: %{error}'
-  identity_proofs:
-    active: Active
-    authorize: Yes, authorize
-    authorize_connection_prompt: Authorize this cryptographic connection?
-    errors:
-      failed: The cryptographic connection failed. Please try again from %{provider}.
-      keybase:
-        invalid_token: Keybase tokens are hashes of signatures and must be 66 hex characters
-        verification_failed: Keybase does not recognize this token as a signature of Keybase user %{kb_username}. Please retry from Keybase.
-      wrong_user: Cannot create a proof for %{proving} while logged in as %{current}. Log in as %{proving} and try again.
-    explanation_html: Here you can cryptographically connect your other identities from other platforms, such as Keybase. This lets other people send you encrypted messages on those platforms and allows them to trust that the content you send them comes from you.
-    i_am_html: I am %{username} on %{service}.
-    identity: Identity
-    inactive: Inactive
-    publicize_checkbox: 'And toot this:'
-    publicize_toot: 'It is proven! I am %{username} on %{service}: %{url}'
-    remove: Remove proof from account
-    removed: Successfully removed proof from account
-    status: Verification status
-    view_proof: View proof
   imports:
     errors:
       over_rows_processing_limit: contains more than %{count} rows
@@ -1279,7 +1259,6 @@ en:
     edit_profile: Edit profile
     export: Data export
     featured_tags: Featured hashtags
-    identity_proofs: Identity proofs
     import: Import
     import_and_export: Import and export
     migrate: Account migration
diff --git a/config/navigation.rb b/config/navigation.rb
index 477d1c9ff..99743c222 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -7,7 +7,6 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url, if: -> { current_user.functional? } do |s|
       s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url
       s.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_url
-      s.item :identity_proofs, safe_join([fa_icon('key fw'), t('settings.identity_proofs')]), settings_identity_proofs_path, highlights_on: %r{/settings/identity_proofs*}, if: proc { current_account.identity_proofs.exists? }
     end
 
     n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_url, if: -> { current_user.functional? } do |s|
diff --git a/config/routes.rb b/config/routes.rb
index c7317d173..5f73129ea 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -25,7 +25,6 @@ Rails.application.routes.draw do
   get '.well-known/nodeinfo', to: 'well_known/nodeinfo#index', as: :nodeinfo, defaults: { format: 'json' }
   get '.well-known/webfinger', to: 'well_known/webfinger#show', as: :webfinger
   get '.well-known/change-password', to: redirect('/auth/edit')
-  get '.well-known/keybase-proof-config', to: 'well_known/keybase_proof_config#show'
 
   get '/nodeinfo/2.0', to: 'well_known/nodeinfo#show', as: :nodeinfo_schema
 
@@ -146,8 +145,6 @@ Rails.application.routes.draw do
       resource :confirmation, only: [:new, :create]
     end
 
-    resources :identity_proofs, only: [:index, :new, :create, :destroy]
-
     resources :applications, except: [:edit] do
       member do
         post :regenerate
@@ -332,9 +329,6 @@ Rails.application.routes.draw do
     # OEmbed
     get '/oembed', to: 'oembed#show', as: :oembed
 
-    # Identity proofs
-    get :proofs, to: 'proofs#index'
-
     # JSON / REST API
     namespace :v1 do
       resources :statuses, only: [:create, :show, :destroy] do
diff --git a/db/post_migrate/20211126000907_drop_account_identity_proofs.rb b/db/post_migrate/20211126000907_drop_account_identity_proofs.rb
new file mode 100644
index 000000000..44a6f1f08
--- /dev/null
+++ b/db/post_migrate/20211126000907_drop_account_identity_proofs.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class DropAccountIdentityProofs < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    drop_table :account_identity_proofs
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 00969daf1..54a46730c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_11_23_212714) do
+ActiveRecord::Schema.define(version: 2021_11_26_000907) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -51,18 +51,6 @@ ActiveRecord::Schema.define(version: 2021_11_23_212714) do
     t.index ["account_id", "domain"], name: "index_account_domain_blocks_on_account_id_and_domain", unique: true
   end
 
-  create_table "account_identity_proofs", force: :cascade do |t|
-    t.bigint "account_id"
-    t.string "provider", default: "", null: false
-    t.string "provider_username", default: "", null: false
-    t.text "token", default: "", null: false
-    t.boolean "verified", default: false, null: false
-    t.boolean "live", default: false, null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.index ["account_id", "provider", "provider_username"], name: "index_account_proofs_on_account_and_provider_and_username", unique: true
-  end
-
   create_table "account_migrations", force: :cascade do |t|
     t.bigint "account_id"
     t.string "acct", default: "", null: false
@@ -1010,7 +998,6 @@ ActiveRecord::Schema.define(version: 2021_11_23_212714) do
   add_foreign_key "account_conversations", "conversations", on_delete: :cascade
   add_foreign_key "account_deletion_requests", "accounts", on_delete: :cascade
   add_foreign_key "account_domain_blocks", "accounts", name: "fk_206c6029bd", on_delete: :cascade
-  add_foreign_key "account_identity_proofs", "accounts", on_delete: :cascade
   add_foreign_key "account_migrations", "accounts", column: "target_account_id", on_delete: :nullify
   add_foreign_key "account_migrations", "accounts", on_delete: :cascade
   add_foreign_key "account_moderation_notes", "accounts"
diff --git a/spec/controllers/api/proofs_controller_spec.rb b/spec/controllers/api/proofs_controller_spec.rb
deleted file mode 100644
index 2fe615005..000000000
--- a/spec/controllers/api/proofs_controller_spec.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-require 'rails_helper'
-
-describe Api::ProofsController do
-  let(:alice) { Fabricate(:account, username: 'alice') }
-
-  before do
-    stub_request(:get, 'https://keybase.io/_/api/1.0/sig/proof_valid.json?domain=cb6e6126.ngrok.io&kb_username=crypto_alice&sig_hash=111111111111111111111111111111111111111111111111111111111111111111&username=alice').to_return(status: 200, body: '{"proof_valid":true,"proof_live":false}')
-    stub_request(:get, 'https://keybase.io/_/api/1.0/sig/proof_live.json?domain=cb6e6126.ngrok.io&kb_username=crypto_alice&sig_hash=111111111111111111111111111111111111111111111111111111111111111111&username=alice').to_return(status: 200, body: '{"proof_valid":true,"proof_live":true}')
-    stub_request(:get, 'https://keybase.io/_/api/1.0/sig/proof_valid.json?domain=cb6e6126.ngrok.io&kb_username=hidden_alice&sig_hash=222222222222222222222222222222222222222222222222222222222222222222&username=alice').to_return(status: 200, body: '{"proof_valid":true,"proof_live":true}')
-    stub_request(:get, 'https://keybase.io/_/api/1.0/sig/proof_live.json?domain=cb6e6126.ngrok.io&kb_username=hidden_alice&sig_hash=222222222222222222222222222222222222222222222222222222222222222222&username=alice').to_return(status: 200, body: '{"proof_valid":true,"proof_live":true}')
-  end
-
-  describe 'GET #index' do
-    describe 'with a non-existent username' do
-      it '404s' do
-        get :index, params: { username: 'nonexistent', provider: 'keybase' }
-
-        expect(response).to have_http_status(:not_found)
-      end
-    end
-
-    describe 'with a user that has no proofs' do
-      it 'is an empty list of signatures' do
-        get :index, params: { username: alice.username, provider: 'keybase' }
-
-        expect(body_as_json[:signatures]).to eq []
-      end
-    end
-
-    describe 'with a user that has a live, valid proof' do
-      let(:token1) { '111111111111111111111111111111111111111111111111111111111111111111' }
-      let(:kb_name1) { 'crypto_alice' }
-
-      before do
-        Fabricate(:account_identity_proof, account: alice, verified: true, live: true, token: token1, provider_username: kb_name1)
-      end
-
-      it 'is a list with that proof in it' do
-        get :index, params: { username: alice.username, provider: 'keybase' }
-
-        expect(body_as_json[:signatures]).to eq [
-          { kb_username: kb_name1, sig_hash: token1 },
-        ]
-      end
-
-      describe 'add one that is neither live nor valid' do
-        let(:token2) { '222222222222222222222222222222222222222222222222222222222222222222' }
-        let(:kb_name2) { 'hidden_alice' }
-
-        before do
-          Fabricate(:account_identity_proof, account: alice, verified: false, live: false, token: token2, provider_username: kb_name2)
-        end
-
-        it 'is a list with both proofs' do
-          get :index, params: { username: alice.username, provider: 'keybase' }
-
-          expect(body_as_json[:signatures]).to eq [
-            { kb_username: kb_name1, sig_hash: token1 },
-            { kb_username: kb_name2, sig_hash: token2 },
-          ]
-        end
-      end
-    end
-
-    describe 'a user that has an avatar' do
-      let(:alice) { Fabricate(:account, username: 'alice', avatar: attachment_fixture('avatar.gif')) }
-
-      context 'and a proof' do
-        let(:token1) { '111111111111111111111111111111111111111111111111111111111111111111' }
-        let(:kb_name1) { 'crypto_alice' }
-
-        before do
-          Fabricate(:account_identity_proof, account: alice, verified: true, live: true, token: token1, provider_username: kb_name1)
-          get :index, params: { username: alice.username, provider: 'keybase' }
-        end
-
-        it 'has two keys: signatures and avatar' do
-          expect(body_as_json.keys).to match_array [:signatures, :avatar]
-        end
-
-        it 'has the correct signatures' do
-          expect(body_as_json[:signatures]).to eq [
-            { kb_username: kb_name1, sig_hash: token1 },
-          ]
-        end
-
-        it 'has the correct avatar url' do
-          expect(body_as_json[:avatar]).to match "https://cb6e6126.ngrok.io#{alice.avatar.url}"
-        end
-      end
-    end
-  end
-end
diff --git a/spec/controllers/settings/identity_proofs_controller_spec.rb b/spec/controllers/settings/identity_proofs_controller_spec.rb
deleted file mode 100644
index 16f236227..000000000
--- a/spec/controllers/settings/identity_proofs_controller_spec.rb
+++ /dev/null
@@ -1,186 +0,0 @@
-require 'rails_helper'
-
-describe Settings::IdentityProofsController do
-  include RoutingHelper
-  render_views
-
-  let(:user) { Fabricate(:user) }
-  let(:valid_token) { '1'*66 }
-  let(:kbname) { 'kbuser' }
-  let(:provider) { 'keybase' }
-  let(:findable_id) { Faker::Number.number(digits: 5) }
-  let(:unfindable_id) { Faker::Number.number(digits: 5) }
-  let(:new_proof_params) do
-    { provider: provider, provider_username: kbname, token: valid_token, username: user.account.username }
-  end
-  let(:status_text) { "i just proved that i am also #{kbname} on #{provider}." }
-  let(:status_posting_params) do
-    { post_status: '0', status_text: status_text }
-  end
-  let(:postable_params) do
-    { account_identity_proof: new_proof_params.merge(status_posting_params) }
-  end
-
-  before do
-    allow_any_instance_of(ProofProvider::Keybase::Verifier).to receive(:status) { { 'proof_valid' => true, 'proof_live' => true } }
-    sign_in user, scope: :user
-  end
-
-  describe 'new proof creation' do
-    context 'GET #new' do
-      before do
-        allow_any_instance_of(ProofProvider::Keybase::Badge).to receive(:avatar_url) { full_pack_url('media/images/void.png') }
-      end
-
-      context 'with all of the correct params' do
-        it 'renders the template' do
-          get :new, params: new_proof_params
-          expect(response).to render_template(:new)
-        end
-      end
-
-      context 'without any params' do
-        it 'redirects to :index' do
-          get :new, params: {}
-          expect(response).to redirect_to settings_identity_proofs_path
-        end
-      end
-
-      context 'with params to prove a different, not logged-in user' do
-        let(:wrong_user_params) { new_proof_params.merge(username: 'someone_else') }
-
-        it 'shows a helpful alert' do
-          get :new, params: wrong_user_params
-          expect(flash[:alert]).to eq I18n.t('identity_proofs.errors.wrong_user', proving: 'someone_else', current: user.account.username)
-        end
-      end
-
-      context 'with params to prove the same username cased differently' do
-        let(:capitalized_username) { new_proof_params.merge(username: user.account.username.upcase) }
-
-        it 'renders the new template' do
-          get :new, params: capitalized_username
-          expect(response).to render_template(:new)
-        end
-      end
-    end
-
-    context 'POST #create' do
-      context 'when saving works' do
-        before do
-          allow(ProofProvider::Keybase::Worker).to receive(:perform_async)
-          allow_any_instance_of(ProofProvider::Keybase::Verifier).to receive(:valid?) { true }
-          allow_any_instance_of(AccountIdentityProof).to receive(:on_success_path) { root_url }
-        end
-
-        it 'serializes a ProofProvider::Keybase::Worker' do
-          expect(ProofProvider::Keybase::Worker).to receive(:perform_async)
-          post :create, params: postable_params
-        end
-
-        it 'delegates redirection to the proof provider' do
-          expect_any_instance_of(AccountIdentityProof).to receive(:on_success_path)
-          post :create, params: postable_params
-          expect(response).to redirect_to root_url
-        end
-
-        it 'does not post a status' do
-          expect(PostStatusService).not_to receive(:new)
-          post :create, params: postable_params
-        end
-
-        context 'and the user has requested to post a status' do
-          let(:postable_params_with_status) do
-            postable_params.tap { |p| p[:account_identity_proof][:post_status] = '1' }
-          end
-
-          it 'posts a status' do
-            expect_any_instance_of(PostStatusService).to receive(:call).with(user.account, text: status_text)
-
-            post :create, params: postable_params_with_status
-          end
-        end
-      end
-
-      context 'when saving fails' do
-        before do
-          allow_any_instance_of(ProofProvider::Keybase::Verifier).to receive(:valid?) { false }
-        end
-
-        it 'redirects to :index' do
-          post :create, params: postable_params
-          expect(response).to redirect_to settings_identity_proofs_path
-        end
-
-        it 'flashes a helpful message' do
-          post :create, params: postable_params
-          expect(flash[:alert]).to eq I18n.t('identity_proofs.errors.failed', provider: 'Keybase')
-        end
-      end
-
-      context 'it can also do an update if the provider and username match an existing proof' do
-        before do
-          allow_any_instance_of(ProofProvider::Keybase::Verifier).to receive(:valid?) { true }
-          allow(ProofProvider::Keybase::Worker).to receive(:perform_async)
-          Fabricate(:account_identity_proof, account: user.account, provider: provider, provider_username: kbname)
-          allow_any_instance_of(AccountIdentityProof).to receive(:on_success_path) { root_url }
-        end
-
-        it 'calls update with the new token' do
-          expect_any_instance_of(AccountIdentityProof).to receive(:save) do |proof|
-            expect(proof.token).to eq valid_token
-          end
-
-          post :create, params: postable_params
-        end
-      end
-    end
-  end
-
-  describe 'GET #index' do
-    context 'with no existing proofs' do
-      it 'shows the helpful explanation' do
-        get :index
-        expect(response.body).to match I18n.t('identity_proofs.explanation_html')
-      end
-    end
-
-    context 'with two proofs' do
-      before do
-        allow_any_instance_of(ProofProvider::Keybase::Verifier).to receive(:valid?) { true }
-        @proof1 = Fabricate(:account_identity_proof, account: user.account)
-        @proof2 = Fabricate(:account_identity_proof, account: user.account)
-        allow_any_instance_of(AccountIdentityProof).to receive(:badge) { double(avatar_url: '', profile_url: '', proof_url: '') }
-        allow_any_instance_of(AccountIdentityProof).to receive(:refresh!) {}
-      end
-
-      it 'has the first proof username on the page' do
-        get :index
-        expect(response.body).to match /#{Regexp.quote(@proof1.provider_username)}/
-      end
-
-      it 'has the second proof username on the page' do
-        get :index
-        expect(response.body).to match /#{Regexp.quote(@proof2.provider_username)}/
-      end
-    end
-  end
-
-  describe 'DELETE #destroy' do
-    before do
-      allow_any_instance_of(ProofProvider::Keybase::Verifier).to receive(:valid?) { true }
-      @proof1 = Fabricate(:account_identity_proof, account: user.account)
-      allow_any_instance_of(AccountIdentityProof).to receive(:badge) { double(avatar_url: '', profile_url: '', proof_url: '') }
-      allow_any_instance_of(AccountIdentityProof).to receive(:refresh!) {}
-      delete :destroy, params: { id: @proof1.id }
-    end
-
-    it 'redirects to :index' do
-      expect(response).to redirect_to settings_identity_proofs_path
-    end
-
-    it 'removes the proof' do
-      expect(AccountIdentityProof.where(id: @proof1.id).count).to eq 0
-    end
-  end
-end
diff --git a/spec/controllers/well_known/keybase_proof_config_controller_spec.rb b/spec/controllers/well_known/keybase_proof_config_controller_spec.rb
deleted file mode 100644
index 00f251c3c..000000000
--- a/spec/controllers/well_known/keybase_proof_config_controller_spec.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-require 'rails_helper'
-
-describe WellKnown::KeybaseProofConfigController, type: :controller do
-  render_views
-
-  describe 'GET #show' do
-    it 'renders json' do
-      get :show
-
-      expect(response).to have_http_status(200)
-      expect(response.media_type).to eq 'application/json'
-      expect { JSON.parse(response.body) }.not_to raise_exception
-    end
-  end
-end
diff --git a/spec/fabricators/account_identity_proof_fabricator.rb b/spec/fabricators/account_identity_proof_fabricator.rb
deleted file mode 100644
index 7b932fa96..000000000
--- a/spec/fabricators/account_identity_proof_fabricator.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-Fabricator(:account_identity_proof) do
-  account
-  provider 'keybase'
-  provider_username { sequence(:provider_username) { |i| "#{Faker::Lorem.characters(number: 15)}" } }
-  token { sequence(:token) { |i| "#{i}#{Faker::Crypto.sha1()*2}"[0..65] } }
-  verified false
-  live false
-end
diff --git a/spec/lib/proof_provider/keybase/verifier_spec.rb b/spec/lib/proof_provider/keybase/verifier_spec.rb
deleted file mode 100644
index 0081a735d..000000000
--- a/spec/lib/proof_provider/keybase/verifier_spec.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-require 'rails_helper'
-
-describe ProofProvider::Keybase::Verifier do
-  let(:my_domain) { Rails.configuration.x.local_domain }
-
-  let(:keybase_proof) do
-    local_proof = AccountIdentityProof.new(
-      provider: 'Keybase',
-      provider_username: 'cryptoalice',
-      token: '11111111111111111111111111'
-    )
-
-    described_class.new('alice', 'cryptoalice', '11111111111111111111111111', my_domain)
-  end
-
-  let(:query_params) do
-    "domain=#{my_domain}&kb_username=cryptoalice&sig_hash=11111111111111111111111111&username=alice"
-  end
-
-  describe '#valid?' do
-    let(:base_url) { 'https://keybase.io/_/api/1.0/sig/proof_valid.json' }
-
-    context 'when valid' do
-      before do
-        json_response_body = '{"status":{"code":0,"name":"OK"},"proof_valid":true}'
-        stub_request(:get, "#{base_url}?#{query_params}").to_return(status: 200, body: json_response_body)
-      end
-
-      it 'calls out to keybase and returns true' do
-        expect(keybase_proof.valid?).to eq true
-      end
-    end
-
-    context 'when invalid' do
-      before do
-        json_response_body = '{"status":{"code":0,"name":"OK"},"proof_valid":false}'
-        stub_request(:get, "#{base_url}?#{query_params}").to_return(status: 200, body: json_response_body)
-      end
-
-      it 'calls out to keybase and returns false' do
-        expect(keybase_proof.valid?).to eq false
-      end
-    end
-
-    context 'with an unexpected api response' do
-      before do
-        json_response_body = '{"status":{"code":100,"desc":"wrong size hex_id","fields":{"sig_hash":"wrong size hex_id"},"name":"INPUT_ERROR"}}'
-        stub_request(:get, "#{base_url}?#{query_params}").to_return(status: 200, body: json_response_body)
-      end
-
-      it 'swallows the error and returns false' do
-        expect(keybase_proof.valid?).to eq false
-      end
-    end
-  end
-
-  describe '#status' do
-    let(:base_url) { 'https://keybase.io/_/api/1.0/sig/proof_live.json' }
-
-    context 'with a normal response' do
-      before do
-        json_response_body = '{"status":{"code":0,"name":"OK"},"proof_live":false,"proof_valid":true}'
-        stub_request(:get, "#{base_url}?#{query_params}").to_return(status: 200, body: json_response_body)
-      end
-
-      it 'calls out to keybase and returns the status fields as proof_valid and proof_live' do
-        expect(keybase_proof.status).to include({ 'proof_valid' => true, 'proof_live' => false })
-      end
-    end
-
-    context 'with an unexpected keybase response' do
-      before do
-        json_response_body = '{"status":{"code":100,"desc":"missing non-optional field sig_hash","fields":{"sig_hash":"missing non-optional field sig_hash"},"name":"INPUT_ERROR"}}'
-        stub_request(:get, "#{base_url}?#{query_params}").to_return(status: 200, body: json_response_body)
-      end
-
-      it 'raises a ProofProvider::Keybase::UnexpectedResponseError' do
-        expect { keybase_proof.status }.to raise_error ProofProvider::Keybase::UnexpectedResponseError
-      end
-    end
-  end
-end
diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb
index 1b1d878a7..7728b9ba8 100644
--- a/spec/services/activitypub/process_account_service_spec.rb
+++ b/spec/services/activitypub/process_account_service_spec.rb
@@ -30,51 +30,6 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
     end
   end
 
-  context 'identity proofs' do
-    let(:payload) do
-      {
-        id: 'https://foo.test',
-        type: 'Actor',
-        inbox: 'https://foo.test/inbox',
-        attachment: [
-          { type: 'IdentityProof', name: 'Alice', signatureAlgorithm: 'keybase', signatureValue: 'a' * 66 },
-        ],
-      }.with_indifferent_access
-    end
-
-    it 'parses out of attachment' do
-      allow(ProofProvider::Keybase::Worker).to receive(:perform_async)
-
-      account = subject.call('alice', 'example.com', payload)
-
-      expect(account.identity_proofs.count).to eq 1
-
-      proof = account.identity_proofs.first
-
-      expect(proof.provider).to eq 'keybase'
-      expect(proof.provider_username).to eq 'Alice'
-      expect(proof.token).to eq 'a' * 66
-    end
-
-    it 'removes no longer present proofs' do
-      allow(ProofProvider::Keybase::Worker).to receive(:perform_async)
-
-      account   = Fabricate(:account, username: 'alice', domain: 'example.com')
-      old_proof = Fabricate(:account_identity_proof, account: account, provider: 'keybase', provider_username: 'Bob', token: 'b' * 66)
-
-      subject.call('alice', 'example.com', payload)
-
-      expect(account.identity_proofs.count).to eq 1
-      expect(account.identity_proofs.find_by(id: old_proof.id)).to be_nil
-    end
-
-    it 'queues a validity check on the proof' do
-      allow(ProofProvider::Keybase::Worker).to receive(:perform_async)
-      account = subject.call('alice', 'example.com', payload)
-      expect(ProofProvider::Keybase::Worker).to have_received(:perform_async)
-    end
-  end
-
   context 'when account is not suspended' do
     let!(:account) { Fabricate(:account, username: 'alice', domain: 'example.com') }
 
-- 
cgit 


From 7f803c41e2ca54b7b787b1f111f91357136c0e68 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 17 Dec 2021 23:01:21 +0100
Subject: Add ability to purge undeliverable domains from admin interface
 (#16686)

* Add ability to purge undeliverable domains from admin interface

* Add tests
---
 app/controllers/admin/instances_controller.rb      |  9 ++++++
 app/helpers/admin/action_logs_helper.rb            |  4 +++
 app/models/admin/action_log_filter.rb              |  1 +
 app/policies/instance_policy.rb                    |  4 +++
 app/services/purge_domain_service.rb               | 10 +++++++
 app/views/admin/instances/show.html.haml           |  2 ++
 app/workers/admin/domain_purge_worker.rb           |  9 ++++++
 config/locales/en.yml                              |  5 ++++
 config/routes.rb                                   |  2 +-
 .../controllers/admin/instances_controller_spec.rb | 35 ++++++++++++++++++----
 spec/policies/instance_policy_spec.rb              |  2 +-
 spec/services/purge_domain_service_spec.rb         | 27 +++++++++++++++++
 spec/workers/admin/domain_purge_worker_spec.rb     | 18 +++++++++++
 13 files changed, 121 insertions(+), 7 deletions(-)
 create mode 100644 app/services/purge_domain_service.rb
 create mode 100644 app/workers/admin/domain_purge_worker.rb
 create mode 100644 spec/services/purge_domain_service_spec.rb
 create mode 100644 spec/workers/admin/domain_purge_worker_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 748c5de5a..306ec1f53 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -14,6 +14,15 @@ module Admin
       authorize :instance, :show?
     end
 
+    def destroy
+      authorize :instance, :destroy?
+
+      Admin::DomainPurgeWorker.perform_async(@instance.domain)
+
+      log_action :destroy, @instance
+      redirect_to admin_instances_path, notice: I18n.t('admin.instances.destroyed_msg', domain: @instance.domain)
+    end
+
     def clear_delivery_errors
       authorize :delivery, :clear_delivery_errors?
 
diff --git a/app/helpers/admin/action_logs_helper.rb b/app/helpers/admin/action_logs_helper.rb
index ae96f7a34..f3aa4be4f 100644
--- a/app/helpers/admin/action_logs_helper.rb
+++ b/app/helpers/admin/action_logs_helper.rb
@@ -31,6 +31,8 @@ module Admin::ActionLogsHelper
       link_to truncate(record.text), edit_admin_announcement_path(record.id)
     when 'IpBlock'
       "#{record.ip}/#{record.ip.prefix} (#{I18n.t("simple_form.labels.ip_block.severities.#{record.severity}")})"
+    when 'Instance'
+      record.domain
     end
   end
 
@@ -54,6 +56,8 @@ module Admin::ActionLogsHelper
       truncate(attributes['text'].is_a?(Array) ? attributes['text'].last : attributes['text'])
     when 'IpBlock'
       "#{attributes['ip']}/#{attributes['ip'].prefix} (#{I18n.t("simple_form.labels.ip_block.severities.#{attributes['severity']}")})"
+    when 'Instance'
+      attributes['domain']
     end
   end
 end
diff --git a/app/models/admin/action_log_filter.rb b/app/models/admin/action_log_filter.rb
index 2af9d7c9c..d1ad46526 100644
--- a/app/models/admin/action_log_filter.rb
+++ b/app/models/admin/action_log_filter.rb
@@ -26,6 +26,7 @@ class Admin::ActionLogFilter
     destroy_domain_allow: { target_type: 'DomainAllow', action: 'destroy' }.freeze,
     destroy_domain_block: { target_type: 'DomainBlock', action: 'destroy' }.freeze,
     destroy_email_domain_block: { target_type: 'EmailDomainBlock', action: 'destroy' }.freeze,
+    destroy_instance: { target_type: 'Instance', action: 'destroy' }.freeze,
     destroy_unavailable_domain: { target_type: 'UnavailableDomain', action: 'destroy' }.freeze,
     destroy_status: { target_type: 'Status', action: 'destroy' }.freeze,
     disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze,
diff --git a/app/policies/instance_policy.rb b/app/policies/instance_policy.rb
index a73823556..801ca162e 100644
--- a/app/policies/instance_policy.rb
+++ b/app/policies/instance_policy.rb
@@ -8,4 +8,8 @@ class InstancePolicy < ApplicationPolicy
   def show?
     admin?
   end
+
+  def destroy?
+    admin?
+  end
 end
diff --git a/app/services/purge_domain_service.rb b/app/services/purge_domain_service.rb
new file mode 100644
index 000000000..e10a8f0c8
--- /dev/null
+++ b/app/services/purge_domain_service.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class PurgeDomainService < BaseService
+  def call(domain)
+    Account.remote.where(domain: domain).reorder(nil).find_each do |account|
+      DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true)
+    end
+    Instance.refresh
+  end
+end
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index d6542ac3e..e520bca0c 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -84,3 +84,5 @@
       = link_to t('admin.instances.delivery.stop'), stop_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button'
     - else
       = link_to t('admin.instances.delivery.restart'), restart_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button'
+    - unless @instance.delivery_failure_tracker.available? && @instance.accounts_count > 0
+      = link_to t('admin.instances.purge'), admin_instance_path(@instance), data: { confirm: t('admin.instances.confirm_purge'), method: :delete }, class: 'button'
diff --git a/app/workers/admin/domain_purge_worker.rb b/app/workers/admin/domain_purge_worker.rb
new file mode 100644
index 000000000..7cba2c89e
--- /dev/null
+++ b/app/workers/admin/domain_purge_worker.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Admin::DomainPurgeWorker
+  include Sidekiq::Worker
+
+  def perform(domain)
+    PurgeDomainService.new.call(domain)
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index e9a0aea54..080b20983 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -240,6 +240,7 @@ en:
         destroy_domain_allow: Delete Domain Allow
         destroy_domain_block: Delete Domain Block
         destroy_email_domain_block: Delete E-mail Domain Block
+        destroy_instance: Purge Domain
         destroy_ip_block: Delete IP rule
         destroy_status: Delete Post
         destroy_unavailable_domain: Delete Unavailable Domain
@@ -287,6 +288,7 @@ en:
         destroy_domain_allow_html: "%{name} disallowed federation with domain %{target}"
         destroy_domain_block_html: "%{name} unblocked domain %{target}"
         destroy_email_domain_block_html: "%{name} unblocked e-mail domain %{target}"
+        destroy_instance_html: "%{name} purged domain %{target}"
         destroy_ip_block_html: "%{name} deleted rule for IP %{target}"
         destroy_status_html: "%{name} removed post by %{target}"
         destroy_unavailable_domain_html: "%{name} resumed delivery to domain %{target}"
@@ -465,6 +467,7 @@ en:
       back_to_limited: Limited
       back_to_warning: Warning
       by_domain: Domain
+      confirm_purge: Are you sure you want to permanently delete data from this domain?
       delivery:
         all: All
         clear: Clear delivery errors
@@ -480,6 +483,7 @@ en:
       delivery_available: Delivery is available
       delivery_error_days: Delivery error days
       delivery_error_hint: If delivery is not possible for %{count} days, it will be automatically marked as undeliverable.
+      destroyed_msg: Data from %{domain} is now queued for imminent deletion.
       empty: No domains found.
       known_accounts:
         one: "%{count} known account"
@@ -490,6 +494,7 @@ en:
         title: Moderation
       private_comment: Private comment
       public_comment: Public comment
+      purge: Purge
       title: Federation
       total_blocked_by_us: Blocked by us
       total_followed_by_them: Followed by them
diff --git a/config/routes.rb b/config/routes.rb
index 31b398e2c..b3b80624d 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -214,7 +214,7 @@ Rails.application.routes.draw do
       end
     end
 
-    resources :instances, only: [:index, :show], constraints: { id: /[^\/]+/ } do
+    resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ } do
       member do
         post :clear_delivery_errors
         post :restart_delivery
diff --git a/spec/controllers/admin/instances_controller_spec.rb b/spec/controllers/admin/instances_controller_spec.rb
index 8c0b309f2..53427b874 100644
--- a/spec/controllers/admin/instances_controller_spec.rb
+++ b/spec/controllers/admin/instances_controller_spec.rb
@@ -3,8 +3,14 @@ require 'rails_helper'
 RSpec.describe Admin::InstancesController, type: :controller do
   render_views
 
+  let(:current_user) { Fabricate(:user, admin: true) }
+
+  let!(:account)     { Fabricate(:account, domain: 'popular') }
+  let!(:account2)    { Fabricate(:account, domain: 'popular') }
+  let!(:account3)    { Fabricate(:account, domain: 'less.popular') }
+
   before do
-    sign_in Fabricate(:user, admin: true), scope: :user
+    sign_in current_user, scope: :user
   end
 
   describe 'GET #index' do
@@ -16,10 +22,6 @@ RSpec.describe Admin::InstancesController, type: :controller do
     end
 
     it 'renders instances' do
-      Fabricate(:account, domain: 'popular')
-      Fabricate(:account, domain: 'popular')
-      Fabricate(:account, domain: 'less.popular')
-
       get :index, params: { page: 2 }
 
       instances = assigns(:instances).to_a
@@ -29,4 +31,27 @@ RSpec.describe Admin::InstancesController, type: :controller do
       expect(response).to have_http_status(200)
     end
   end
+
+  describe 'DELETE #destroy' do
+    subject { delete :destroy, params: { id: Instance.first.id } }
+
+    let(:current_user) { Fabricate(:user, admin: admin) }
+    let(:account) { Fabricate(:account) }
+
+    context 'when user is admin' do
+      let(:admin) { true }
+
+      it 'succeeds in purging instance' do
+        is_expected.to redirect_to admin_instances_path
+      end
+    end
+
+    context 'when user is not admin' do
+      let(:admin) { false }
+
+      it 'fails to purge instance' do
+        is_expected.to have_http_status :forbidden
+      end
+    end
+  end
 end
diff --git a/spec/policies/instance_policy_spec.rb b/spec/policies/instance_policy_spec.rb
index 77a3bde3f..72cf25f56 100644
--- a/spec/policies/instance_policy_spec.rb
+++ b/spec/policies/instance_policy_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe InstancePolicy do
   let(:admin)   { Fabricate(:user, admin: true).account }
   let(:john)    { Fabricate(:user).account }
 
-  permissions :index? do
+  permissions :index?, :show?, :destroy? do
     context 'admin' do
       it 'permits' do
         expect(subject).to permit(admin, Instance)
diff --git a/spec/services/purge_domain_service_spec.rb b/spec/services/purge_domain_service_spec.rb
new file mode 100644
index 000000000..59285f126
--- /dev/null
+++ b/spec/services/purge_domain_service_spec.rb
@@ -0,0 +1,27 @@
+require 'rails_helper'
+
+RSpec.describe PurgeDomainService, type: :service do
+  let!(:old_account) { Fabricate(:account, domain: 'obsolete.org') }
+  let!(:old_status1) { Fabricate(:status, account: old_account) }
+  let!(:old_status2) { Fabricate(:status, account: old_account) }
+  let!(:old_attachment) { Fabricate(:media_attachment, account: old_account, status: old_status2, file: attachment_fixture('attachment.jpg')) }
+
+  subject { PurgeDomainService.new }
+
+  describe 'for a suspension' do
+    before do
+      subject.call('obsolete.org')
+    end
+
+    it 'removes the remote accounts\'s statuses and media attachments' do
+      expect { old_account.reload }.to raise_exception ActiveRecord::RecordNotFound
+      expect { old_status1.reload }.to raise_exception ActiveRecord::RecordNotFound
+      expect { old_status2.reload }.to raise_exception ActiveRecord::RecordNotFound
+      expect { old_attachment.reload }.to raise_exception ActiveRecord::RecordNotFound
+    end
+
+    it 'refreshes instances view' do
+      expect(Instance.where(domain: 'obsolete.org').exists?).to be false
+    end
+  end
+end
diff --git a/spec/workers/admin/domain_purge_worker_spec.rb b/spec/workers/admin/domain_purge_worker_spec.rb
new file mode 100644
index 000000000..b67c58b23
--- /dev/null
+++ b/spec/workers/admin/domain_purge_worker_spec.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Admin::DomainPurgeWorker do
+  subject { described_class.new }
+
+  describe 'perform' do
+    it 'calls domain purge service for relevant domain block' do
+      service = double(call: nil)
+      allow(PurgeDomainService).to receive(:new).and_return(service)
+      result = subject.perform('example.com')
+
+      expect(result).to be_nil
+      expect(service).to have_received(:call).with('example.com')
+    end
+  end
+end
-- 
cgit 


From d3db2eb7fb17e94ace3fc648153f14178e707830 Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Thu, 30 Dec 2021 16:41:09 +0900
Subject: Remove custom emojis on domain purge (#17210)

---
 app/services/purge_domain_service.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'app/services')

diff --git a/app/services/purge_domain_service.rb b/app/services/purge_domain_service.rb
index e10a8f0c8..9df81f13e 100644
--- a/app/services/purge_domain_service.rb
+++ b/app/services/purge_domain_service.rb
@@ -5,6 +5,7 @@ class PurgeDomainService < BaseService
     Account.remote.where(domain: domain).reorder(nil).find_each do |account|
       DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true)
     end
+    CustomEmoji.remote.where(domain: domain).reorder(nil).find_each(&:destroy)
     Instance.refresh
   end
 end
-- 
cgit 


From d5c9feb7b7fc489afbd0a287431fe07b42451ef0 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 17 Jan 2022 00:49:55 +0100
Subject: Add support for private pinned posts (#16954)

* Add support for private pinned toots

* Allow local user to pin private toots

* Change wording to avoid "direct message"
---
 app/controllers/accounts_controller.rb             | 13 ++++++-
 .../activitypub/collections_controller.rb          |  1 +
 .../api/v1/accounts/statuses_controller.rb         |  4 +--
 .../mastodon/components/status_action_bar.js       |  3 +-
 .../features/status/components/action_bar.js       |  3 +-
 app/lib/activitypub/activity/accept.rb             | 13 +++++--
 app/lib/activitypub/activity/add.rb                |  3 +-
 .../fetch_featured_collection_service.rb           |  6 +++-
 app/validators/status_pin_validator.rb             |  2 +-
 app/workers/remote_account_refresh_worker.rb       | 24 +++++++++++++
 config/locales/en.yml                              |  2 +-
 spec/controllers/accounts_controller_spec.rb       |  1 +
 .../activitypub/collections_controller_spec.rb     | 23 ++++++++++--
 .../api/v1/accounts/statuses_controller_spec.rb    | 35 +++++++++++++++++-
 spec/lib/activitypub/activity/accept_spec.rb       |  5 +++
 spec/lib/activitypub/activity/add_spec.rb          | 42 ++++++++++++++++++----
 spec/models/status_pin_spec.rb                     |  4 +--
 spec/validators/status_pin_validator_spec.rb       |  8 ++---
 18 files changed, 164 insertions(+), 28 deletions(-)
 create mode 100644 app/workers/remote_account_refresh_worker.rb

(limited to 'app/services')

diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 8210918d8..ddd38cbb0 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -28,7 +28,7 @@ class AccountsController < ApplicationController
           return
         end
 
-        @pinned_statuses = cache_collection(@account.pinned_statuses, Status) if show_pinned_statuses?
+        @pinned_statuses = cached_filtered_status_pins if show_pinned_statuses?
         @statuses        = cached_filtered_status_page
         @rss_url         = rss_url
 
@@ -64,6 +64,10 @@ class AccountsController < ApplicationController
     [replies_requested?, media_requested?, tag_requested?, params[:max_id].present?, params[:min_id].present?].none?
   end
 
+  def filtered_pinned_statuses
+    @account.pinned_statuses.where(visibility: [:public, :unlisted])
+  end
+
   def filtered_statuses
     default_statuses.tap do |statuses|
       statuses.merge!(hashtag_scope)    if tag_requested?
@@ -142,6 +146,13 @@ class AccountsController < ApplicationController
     request.path.split('.').first.end_with?(Addressable::URI.parse("/tagged/#{params[:tag]}").normalize)
   end
 
+  def cached_filtered_status_pins
+    cache_collection(
+      filtered_pinned_statuses,
+      Status
+    )
+  end
+
   def cached_filtered_status_page
     cache_collection_paginated_by_id(
       filtered_statuses,
diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb
index c8b6dcc88..e4e994a98 100644
--- a/app/controllers/activitypub/collections_controller.rb
+++ b/app/controllers/activitypub/collections_controller.rb
@@ -21,6 +21,7 @@ class ActivityPub::CollectionsController < ActivityPub::BaseController
     case params[:id]
     when 'featured'
       @items = for_signed_account { cache_collection(@account.pinned_statuses, Status) }
+      @items = @items.map { |item| item.distributable? ? item : ActivityPub::TagManager.instance.uri_for(item) }
     when 'tags'
       @items = for_signed_account { @account.featured_tags }
     when 'devices'
diff --git a/app/controllers/api/v1/accounts/statuses_controller.rb b/app/controllers/api/v1/accounts/statuses_controller.rb
index 92ccb8061..2c027ea76 100644
--- a/app/controllers/api/v1/accounts/statuses_controller.rb
+++ b/app/controllers/api/v1/accounts/statuses_controller.rb
@@ -46,9 +46,7 @@ class Api::V1::Accounts::StatusesController < Api::BaseController
   end
 
   def pinned_scope
-    return Status.none if @account.blocking?(current_account)
-
-    @account.pinned_statuses
+    @account.pinned_statuses.permitted_for(@account, current_account)
   end
 
   def no_replies_scope
diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js
index 85c76edee..d125359e9 100644
--- a/app/javascript/mastodon/components/status_action_bar.js
+++ b/app/javascript/mastodon/components/status_action_bar.js
@@ -225,6 +225,7 @@ class StatusActionBar extends ImmutablePureComponent {
 
     const anonymousAccess    = !me;
     const publicStatus       = ['public', 'unlisted'].includes(status.get('visibility'));
+    const pinnableStatus     = ['public', 'unlisted', 'private'].includes(status.get('visibility'));
     const mutingConversation = status.get('muted');
     const account            = status.get('account');
     const writtenByMe        = status.getIn(['account', 'id']) === me;
@@ -242,7 +243,7 @@ class StatusActionBar extends ImmutablePureComponent {
 
     menu.push({ text: intl.formatMessage(status.get('bookmarked') ? messages.removeBookmark : messages.bookmark), action: this.handleBookmarkClick });
 
-    if (writtenByMe && publicStatus) {
+    if (writtenByMe && pinnableStatus) {
       menu.push({ text: intl.formatMessage(status.get('pinned') ? messages.unpin : messages.pin), action: this.handlePinClick });
     }
 
diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js
index ffa2510c0..e60119bc4 100644
--- a/app/javascript/mastodon/features/status/components/action_bar.js
+++ b/app/javascript/mastodon/features/status/components/action_bar.js
@@ -188,6 +188,7 @@ class ActionBar extends React.PureComponent {
     const { status, relationship, intl } = this.props;
 
     const publicStatus       = ['public', 'unlisted'].includes(status.get('visibility'));
+    const pinnableStatus     = ['public', 'unlisted', 'private'].includes(status.get('visibility'));
     const mutingConversation = status.get('muted');
     const account            = status.get('account');
     const writtenByMe        = status.getIn(['account', 'id']) === me;
@@ -201,7 +202,7 @@ class ActionBar extends React.PureComponent {
     }
 
     if (writtenByMe) {
-      if (publicStatus) {
+      if (pinnableStatus) {
         menu.push({ text: intl.formatMessage(status.get('pinned') ? messages.unpin : messages.pin), action: this.handlePinClick });
         menu.push(null);
       }
diff --git a/app/lib/activitypub/activity/accept.rb b/app/lib/activitypub/activity/accept.rb
index 7010ff43e..5126e23c6 100644
--- a/app/lib/activitypub/activity/accept.rb
+++ b/app/lib/activitypub/activity/accept.rb
@@ -3,7 +3,7 @@
 class ActivityPub::Activity::Accept < ActivityPub::Activity
   def perform
     return accept_follow_for_relay if relay_follow?
-    return follow_request_from_object.authorize! unless follow_request_from_object.nil?
+    return accept_follow!(follow_request_from_object) unless follow_request_from_object.nil?
 
     case @object['type']
     when 'Follow'
@@ -19,7 +19,16 @@ class ActivityPub::Activity::Accept < ActivityPub::Activity
     return if target_account.nil? || !target_account.local?
 
     follow_request = FollowRequest.find_by(account: target_account, target_account: @account)
-    follow_request&.authorize!
+    accept_follow!(follow_request)
+  end
+
+  def accept_follow!(request)
+    return if request.nil?
+
+    is_first_follow = !request.target_account.followers.local.exists?
+    request.authorize!
+
+    RemoteAccountRefreshWorker.perform_async(request.target_account_id) if is_first_follow
   end
 
   def accept_follow_for_relay
diff --git a/app/lib/activitypub/activity/add.rb b/app/lib/activitypub/activity/add.rb
index 688ab00b3..845eeaef7 100644
--- a/app/lib/activitypub/activity/add.rb
+++ b/app/lib/activitypub/activity/add.rb
@@ -4,8 +4,7 @@ class ActivityPub::Activity::Add < ActivityPub::Activity
   def perform
     return unless @json['target'].present? && value_or_id(@json['target']) == @account.featured_collection_url
 
-    status   = status_from_uri(object_uri)
-    status ||= fetch_remote_original_status
+    status = status_from_object
 
     return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
 
diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index 72352aca6..9fce478c1 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -23,7 +23,7 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
 
   def process_items(items)
     status_ids = items.map { |item| value_or_id(item) }
-                      .filter_map { |uri| ActivityPub::FetchRemoteStatusService.new.call(uri) unless ActivityPub::TagManager.instance.local_uri?(uri) }
+                      .filter_map { |uri| ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower) unless ActivityPub::TagManager.instance.local_uri?(uri) }
                       .filter_map { |status| status.id if status.account_id == @account.id }
     to_remove = []
     to_add    = status_ids
@@ -46,4 +46,8 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
   def supported_context?
     super(@json)
   end
+
+  def local_follower
+    @local_follower ||= account.followers.local.without_suspended.first
+  end
 end
diff --git a/app/validators/status_pin_validator.rb b/app/validators/status_pin_validator.rb
index 2c7bce674..2fdd5b34f 100644
--- a/app/validators/status_pin_validator.rb
+++ b/app/validators/status_pin_validator.rb
@@ -4,7 +4,7 @@ class StatusPinValidator < ActiveModel::Validator
   def validate(pin)
     pin.errors.add(:base, I18n.t('statuses.pin_errors.reblog')) if pin.status.reblog?
     pin.errors.add(:base, I18n.t('statuses.pin_errors.ownership')) if pin.account_id != pin.status.account_id
-    pin.errors.add(:base, I18n.t('statuses.pin_errors.private')) unless %w(public unlisted).include?(pin.status.visibility)
+    pin.errors.add(:base, I18n.t('statuses.pin_errors.direct')) if pin.status.direct_visibility?
     pin.errors.add(:base, I18n.t('statuses.pin_errors.limit')) if pin.account.status_pins.count > 4 && pin.account.local?
   end
 end
diff --git a/app/workers/remote_account_refresh_worker.rb b/app/workers/remote_account_refresh_worker.rb
new file mode 100644
index 000000000..9632936b5
--- /dev/null
+++ b/app/workers/remote_account_refresh_worker.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class RemoteAccountRefreshWorker
+  include Sidekiq::Worker
+  include ExponentialBackoff
+  include JsonLdHelper
+
+  sidekiq_options queue: 'pull', retry: 3
+
+  def perform(id)
+    account = Account.find_by(id: id)
+    return if account.nil? || account.local?
+
+    ActivityPub::FetchRemoteAccountService.new.call(account.uri)
+  rescue Mastodon::UnexpectedResponseError => e
+    response = e.response
+
+    if response_error_unsalvageable?(response)
+      # Give up
+    else
+      raise e
+    end
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 32b48dbff..693a7b400 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1300,9 +1300,9 @@ en:
     open_in_web: Open in web
     over_character_limit: character limit of %{max} exceeded
     pin_errors:
+      direct: Posts that are only visible to mentioned users cannot be pinned
       limit: You have already pinned the maximum number of posts
       ownership: Someone else's post cannot be pinned
-      private: Non-public posts cannot be pinned
       reblog: A boost cannot be pinned
     poll:
       total_people:
diff --git a/spec/controllers/accounts_controller_spec.rb b/spec/controllers/accounts_controller_spec.rb
index ac426b01e..7c5ba8754 100644
--- a/spec/controllers/accounts_controller_spec.rb
+++ b/spec/controllers/accounts_controller_spec.rb
@@ -35,6 +35,7 @@ RSpec.describe AccountsController, type: :controller do
     before do
       status_media.media_attachments << Fabricate(:media_attachment, account: account, type: :image)
       account.pinned_statuses << status_pinned
+      account.pinned_statuses << status_private
     end
 
     shared_examples 'preliminary checks' do
diff --git a/spec/controllers/activitypub/collections_controller_spec.rb b/spec/controllers/activitypub/collections_controller_spec.rb
index d584136ff..21a033945 100644
--- a/spec/controllers/activitypub/collections_controller_spec.rb
+++ b/spec/controllers/activitypub/collections_controller_spec.rb
@@ -4,6 +4,7 @@ require 'rails_helper'
 
 RSpec.describe ActivityPub::CollectionsController, type: :controller do
   let!(:account) { Fabricate(:account) }
+  let!(:private_pinned) { Fabricate(:status, account: account, text: 'secret private stuff', visibility: :private) }
   let(:remote_account) { nil }
 
   shared_examples 'cachable response' do
@@ -27,6 +28,7 @@ RSpec.describe ActivityPub::CollectionsController, type: :controller do
 
     Fabricate(:status_pin, account: account)
     Fabricate(:status_pin, account: account)
+    Fabricate(:status_pin, account: account, status: private_pinned)
     Fabricate(:status, account: account, visibility: :private)
   end
 
@@ -50,7 +52,15 @@ RSpec.describe ActivityPub::CollectionsController, type: :controller do
 
         it 'returns orderedItems with pinned statuses' do
           expect(body[:orderedItems]).to be_an Array
-          expect(body[:orderedItems].size).to eq 2
+          expect(body[:orderedItems].size).to eq 3
+        end
+
+        it 'includes URI of private pinned status' do
+          expect(body[:orderedItems]).to include(ActivityPub::TagManager.instance.uri_for(private_pinned))
+        end
+
+        it 'does not include contents of private pinned status' do
+          expect(response.body).not_to include(private_pinned.text)
         end
 
         context 'when account is permanently suspended' do
@@ -96,7 +106,16 @@ RSpec.describe ActivityPub::CollectionsController, type: :controller do
           it 'returns orderedItems with pinned statuses' do
             json = body_as_json
             expect(json[:orderedItems]).to be_an Array
-            expect(json[:orderedItems].size).to eq 2
+            expect(json[:orderedItems].size).to eq 3
+          end
+
+          it 'includes URI of private pinned status' do
+            json = body_as_json
+            expect(json[:orderedItems]).to include(ActivityPub::TagManager.instance.uri_for(private_pinned))
+          end
+
+          it 'does not include contents of private pinned status' do
+            expect(response.body).not_to include(private_pinned.text)
           end
         end
 
diff --git a/spec/controllers/api/v1/accounts/statuses_controller_spec.rb b/spec/controllers/api/v1/accounts/statuses_controller_spec.rb
index 693cd1ac6..0a18ddcbd 100644
--- a/spec/controllers/api/v1/accounts/statuses_controller_spec.rb
+++ b/spec/controllers/api/v1/accounts/statuses_controller_spec.rb
@@ -39,7 +39,7 @@ describe Api::V1::Accounts::StatusesController do
       end
     end
 
-    context 'with only pinned' do
+    context 'with only own pinned' do
       before do
         Fabricate(:status_pin, account: user.account, status: Fabricate(:status, account: user.account))
       end
@@ -50,5 +50,38 @@ describe Api::V1::Accounts::StatusesController do
         expect(response).to have_http_status(200)
       end
     end
+
+    context "with someone else's pinned statuses" do
+      let(:account)        { Fabricate(:account, username: 'bob', domain: 'example.com') }
+      let(:status)         { Fabricate(:status, account: account) }
+      let(:private_status) { Fabricate(:status, account: account, visibility: :private) }
+      let!(:pin)           { Fabricate(:status_pin, account: account, status: status) }
+      let!(:private_pin)   { Fabricate(:status_pin, account: account, status: private_status) }
+
+      it 'returns http success' do
+        get :index, params: { account_id: account.id, pinned: true }
+        expect(response).to have_http_status(200)
+      end
+
+      context 'when user does not follow account' do
+        it 'lists the public status only' do
+          get :index, params: { account_id: account.id, pinned: true }
+          json = body_as_json
+          expect(json.map { |item| item[:id].to_i }).to eq [status.id]
+        end
+      end
+
+      context 'when user follows account' do
+        before do
+          user.account.follow!(account)
+        end
+
+        it 'lists both the public and the private statuses' do
+          get :index, params: { account_id: account.id, pinned: true }
+          json = body_as_json
+          expect(json.map { |item| item[:id].to_i }.sort).to eq [status.id, private_status.id].sort
+        end
+      end
+    end
   end
 end
diff --git a/spec/lib/activitypub/activity/accept_spec.rb b/spec/lib/activitypub/activity/accept_spec.rb
index 883bab6ac..304cf2208 100644
--- a/spec/lib/activitypub/activity/accept_spec.rb
+++ b/spec/lib/activitypub/activity/accept_spec.rb
@@ -23,6 +23,7 @@ RSpec.describe ActivityPub::Activity::Accept do
     subject { described_class.new(json, sender) }
 
     before do
+      allow(RemoteAccountRefreshWorker).to receive(:perform_async)
       Fabricate(:follow_request, account: recipient, target_account: sender)
       subject.perform
     end
@@ -34,6 +35,10 @@ RSpec.describe ActivityPub::Activity::Accept do
     it 'removes the follow request' do
       expect(recipient.requested?(sender)).to be false
     end
+
+    it 'queues a refresh' do
+      expect(RemoteAccountRefreshWorker).to have_received(:perform_async).with(sender.id)
+    end
   end
 
   context 'given a relay' do
diff --git a/spec/lib/activitypub/activity/add_spec.rb b/spec/lib/activitypub/activity/add_spec.rb
index 16db71c88..e6408b610 100644
--- a/spec/lib/activitypub/activity/add_spec.rb
+++ b/spec/lib/activitypub/activity/add_spec.rb
@@ -1,8 +1,8 @@
 require 'rails_helper'
 
 RSpec.describe ActivityPub::Activity::Add do
-  let(:sender) { Fabricate(:account, featured_collection_url: 'https://example.com/featured') }
-  let(:status) { Fabricate(:status, account: sender) }
+  let(:sender) { Fabricate(:account, featured_collection_url: 'https://example.com/featured', domain: 'example.com') }
+  let(:status) { Fabricate(:status, account: sender, visibility: :private) }
 
   let(:json) do
     {
@@ -24,6 +24,8 @@ RSpec.describe ActivityPub::Activity::Add do
     end
 
     context 'when status was not known before' do
+      let(:service_stub) { double }
+
       let(:json) do
         {
           '@context': 'https://www.w3.org/ns/activitystreams',
@@ -36,12 +38,40 @@ RSpec.describe ActivityPub::Activity::Add do
       end
 
       before do
-        stub_request(:get, 'https://example.com/unknown').to_return(status: 410)
+        allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service_stub)
+      end
+
+      context 'when there is a local follower' do
+        before do
+          account = Fabricate(:account)
+          account.follow!(sender)
+        end
+
+        it 'fetches the status and pins it' do
+          allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil|
+            expect(uri).to eq 'https://example.com/unknown'
+            expect(id).to eq true
+            expect(on_behalf_of&.following?(sender)).to eq true
+            status
+          end
+          subject.perform
+          expect(service_stub).to have_received(:call)
+          expect(sender.pinned?(status)).to be true
+        end
       end
 
-      it 'fetches the status' do
-        subject.perform
-        expect(a_request(:get, 'https://example.com/unknown')).to have_been_made.at_least_once
+      context 'when there is no local follower' do
+        it 'tries to fetch the status' do
+          allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil|
+            expect(uri).to eq 'https://example.com/unknown'
+            expect(id).to eq true
+            expect(on_behalf_of).to eq nil
+            nil
+          end
+          subject.perform
+          expect(service_stub).to have_received(:call)
+          expect(sender.pinned?(status)).to be false
+        end
       end
     end
   end
diff --git a/spec/models/status_pin_spec.rb b/spec/models/status_pin_spec.rb
index 6f0b2feb8..c18faca78 100644
--- a/spec/models/status_pin_spec.rb
+++ b/spec/models/status_pin_spec.rb
@@ -24,11 +24,11 @@ RSpec.describe StatusPin, type: :model do
       expect(StatusPin.new(account: account, status: reblog).save).to be false
     end
 
-    it 'does not allow pins of private statuses' do
+    it 'does allow pins of direct statuses' do
       account = Fabricate(:account)
       status  = Fabricate(:status, account: account, visibility: :private)
 
-      expect(StatusPin.new(account: account, status: status).save).to be false
+      expect(StatusPin.new(account: account, status: status).save).to be true
     end
 
     it 'does not allow pins of direct statuses' do
diff --git a/spec/validators/status_pin_validator_spec.rb b/spec/validators/status_pin_validator_spec.rb
index 06532e5b3..d5bd0d1b8 100644
--- a/spec/validators/status_pin_validator_spec.rb
+++ b/spec/validators/status_pin_validator_spec.rb
@@ -9,7 +9,7 @@ RSpec.describe StatusPinValidator, type: :validator do
     end
 
     let(:pin) { double(account: account, errors: errors, status: status, account_id: pin_account_id) }
-    let(:status) { double(reblog?: reblog, account_id: status_account_id, visibility: visibility) }
+    let(:status) { double(reblog?: reblog, account_id: status_account_id, visibility: visibility, direct_visibility?: visibility == 'direct') }
     let(:account)     { double(status_pins: status_pins, local?: local) }
     let(:status_pins) { double(count: count) }
     let(:errors)      { double(add: nil) }
@@ -37,11 +37,11 @@ RSpec.describe StatusPinValidator, type: :validator do
       end
     end
 
-    context 'unless %w(public unlisted).include?(pin.status.visibility)' do
-      let(:visibility) { '' }
+    context 'if pin.status.direct_visibility?' do
+      let(:visibility) { 'direct' }
 
       it 'calls errors.add' do
-        expect(errors).to have_received(:add).with(:base, I18n.t('statuses.pin_errors.private'))
+        expect(errors).to have_received(:add).with(:base, I18n.t('statuses.pin_errors.direct'))
       end
     end
 
-- 
cgit 


From 14f436c457560862fafabd753eb314c8b8a8e674 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 17 Jan 2022 09:41:33 +0100
Subject: Add notifications for statuses deleted by moderators (#17204)

---
 .../admin/account_moderation_notes_controller.rb   |   2 +-
 app/controllers/admin/accounts_controller.rb       |   2 +-
 app/controllers/admin/report_notes_controller.rb   |  23 +-
 .../admin/reported_statuses_controller.rb          |  44 ---
 app/controllers/admin/reports_controller.rb        |   6 +-
 app/controllers/admin/statuses_controller.rb       |  66 ++---
 .../api/v1/admin/account_actions_controller.rb     |   4 +-
 .../api/v1/admin/accounts_controller.rb            |   6 +-
 .../api/v1/admin/dimensions_controller.rb          |   1 +
 .../api/v1/admin/measures_controller.rb            |   1 +
 app/controllers/api/v1/admin/reports_controller.rb |  16 +-
 .../api/v1/admin/retention_controller.rb           |   1 +
 .../api/v1/admin/trends/tags_controller.rb         |   3 +
 app/helpers/admin/filter_helper.rb                 |   1 +
 .../components/admin/ReportReasonSelector.js       | 159 ++++++++++
 .../mastodon/components/status_action_bar.js       |   2 +-
 .../features/status/components/action_bar.js       |   2 +-
 app/javascript/styles/mailer.scss                  |   4 +
 app/javascript/styles/mastodon/admin.scss          | 328 ++++++++++++++++++++-
 app/javascript/styles/mastodon/polls.scss          |  15 +
 .../metrics/measure/resolved_reports_measure.rb    |   7 +-
 app/mailers/user_mailer.rb                         |   4 +-
 app/models/account_warning.rb                      |  22 +-
 app/models/admin/account_action.rb                 |  28 +-
 app/models/admin/status_batch_action.rb            |  92 ++++++
 app/models/admin/status_filter.rb                  |  41 +++
 app/models/concerns/account_associations.rb        |   2 +-
 app/models/form/status_batch.rb                    |  45 ---
 app/models/report.rb                               |  66 +++--
 app/models/report_filter.rb                        |   2 +-
 app/serializers/rest/admin/report_serializer.rb    |   7 +-
 app/services/remove_status_service.rb              |   9 +-
 app/views/admin/action_logs/index.html.haml        |   2 +-
 .../admin/report_notes/_report_note.html.haml      |  23 +-
 app/views/admin/reports/_action_log.html.haml      |   6 -
 app/views/admin/reports/_status.html.haml          |   3 +
 app/views/admin/reports/show.html.haml             | 274 +++++++++++------
 app/views/admin/statuses/index.html.haml           |  33 ++-
 app/views/admin/statuses/show.html.haml            |  27 --
 app/views/notification_mailer/_status.text.erb     |   8 +-
 app/views/user_mailer/warning.html.haml            |  16 +-
 app/views/user_mailer/warning.text.erb             |  17 +-
 app/workers/scheduler/user_cleanup_scheduler.rb    |   9 +
 config/locales/en.yml                              |  55 +++-
 config/routes.rb                                   |  12 +-
 .../20211231080958_add_category_to_reports.rb      |  21 ++
 ...0115125126_add_report_id_to_account_warnings.rb |   6 +
 .../20220115125341_fix_account_warning_actions.rb  |  21 ++
 ...20116202951_add_deleted_at_index_on_statuses.rb |   7 +
 ...20109213908_remove_action_taken_from_reports.rb |   9 +
 db/schema.rb                                       |  10 +-
 .../admin/report_notes_controller_spec.rb          |   8 +-
 .../admin/reported_statuses_controller_spec.rb     |  59 ----
 spec/controllers/admin/reports_controller_spec.rb  |  22 +-
 spec/controllers/admin/statuses_controller_spec.rb |  69 ++---
 spec/fabricators/report_fabricator.rb              |   6 +-
 spec/mailers/previews/user_mailer_preview.rb       |   2 +-
 spec/models/form/status_batch_spec.rb              |  52 ----
 spec/models/report_spec.rb                         |  16 +-
 59 files changed, 1213 insertions(+), 591 deletions(-)
 delete mode 100644 app/controllers/admin/reported_statuses_controller.rb
 create mode 100644 app/javascript/mastodon/components/admin/ReportReasonSelector.js
 create mode 100644 app/models/admin/status_batch_action.rb
 create mode 100644 app/models/admin/status_filter.rb
 delete mode 100644 app/models/form/status_batch.rb
 delete mode 100644 app/views/admin/reports/_action_log.html.haml
 delete mode 100644 app/views/admin/statuses/show.html.haml
 create mode 100644 db/migrate/20211231080958_add_category_to_reports.rb
 create mode 100644 db/migrate/20220115125126_add_report_id_to_account_warnings.rb
 create mode 100644 db/migrate/20220115125341_fix_account_warning_actions.rb
 create mode 100644 db/migrate/20220116202951_add_deleted_at_index_on_statuses.rb
 create mode 100644 db/post_migrate/20220109213908_remove_action_taken_from_reports.rb
 delete mode 100644 spec/controllers/admin/reported_statuses_controller_spec.rb
 delete mode 100644 spec/models/form/status_batch_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/admin/account_moderation_notes_controller.rb b/app/controllers/admin/account_moderation_notes_controller.rb
index 44f6e34f8..4f36f33f4 100644
--- a/app/controllers/admin/account_moderation_notes_controller.rb
+++ b/app/controllers/admin/account_moderation_notes_controller.rb
@@ -14,7 +14,7 @@ module Admin
       else
         @account          = @account_moderation_note.target_account
         @moderation_notes = @account.targeted_moderation_notes.latest
-        @warnings         = @account.targeted_account_warnings.latest.custom
+        @warnings         = @account.strikes.custom.latest
 
         render template: 'admin/accounts/show'
       end
diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index 0786985fa..e7f56e243 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -28,7 +28,7 @@ module Admin
       @deletion_request        = @account.deletion_request
       @account_moderation_note = current_account.account_moderation_notes.new(target_account: @account)
       @moderation_notes        = @account.targeted_moderation_notes.latest
-      @warnings                = @account.targeted_account_warnings.latest.custom
+      @warnings                = @account.strikes.custom.latest
       @domain_block            = DomainBlock.rule_for(@account.domain)
     end
 
diff --git a/app/controllers/admin/report_notes_controller.rb b/app/controllers/admin/report_notes_controller.rb
index b816c5b5d..3fd815b60 100644
--- a/app/controllers/admin/report_notes_controller.rb
+++ b/app/controllers/admin/report_notes_controller.rb
@@ -14,20 +14,17 @@ module Admin
         if params[:create_and_resolve]
           @report.resolve!(current_account)
           log_action :resolve, @report
-
-          redirect_to admin_reports_path, notice: I18n.t('admin.reports.resolved_msg')
-          return
-        end
-
-        if params[:create_and_unresolve]
+        elsif params[:create_and_unresolve]
           @report.unresolve!
           log_action :reopen, @report
         end
 
-        redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg')
+        redirect_to after_create_redirect_path, notice: I18n.t('admin.report_notes.created_msg')
       else
-        @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at)
-        @form         = Form::StatusBatch.new
+        @report_notes = @report.notes.includes(:account).order(id: :desc)
+        @action_logs  = @report.history.includes(:target)
+        @form         = Admin::StatusBatchAction.new
+        @statuses     = @report.statuses.with_includes
 
         render template: 'admin/reports/show'
       end
@@ -41,6 +38,14 @@ module Admin
 
     private
 
+    def after_create_redirect_path
+      if params[:create_and_resolve]
+        admin_reports_path
+      else
+        admin_report_path(@report)
+      end
+    end
+
     def resource_params
       params.require(:report_note).permit(
         :content,
diff --git a/app/controllers/admin/reported_statuses_controller.rb b/app/controllers/admin/reported_statuses_controller.rb
deleted file mode 100644
index 3ba9f5df2..000000000
--- a/app/controllers/admin/reported_statuses_controller.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Admin
-  class ReportedStatusesController < BaseController
-    before_action :set_report
-
-    def create
-      authorize :status, :update?
-
-      @form         = Form::StatusBatch.new(form_status_batch_params.merge(current_account: current_account, action: action_from_button))
-      flash[:alert] = I18n.t('admin.statuses.failed_to_execute') unless @form.save
-
-      redirect_to admin_report_path(@report)
-    rescue ActionController::ParameterMissing
-      flash[:alert] = I18n.t('admin.statuses.no_status_selected')
-
-      redirect_to admin_report_path(@report)
-    end
-
-    private
-
-    def status_params
-      params.require(:status).permit(:sensitive)
-    end
-
-    def form_status_batch_params
-      params.require(:form_status_batch).permit(status_ids: [])
-    end
-
-    def action_from_button
-      if params[:nsfw_on]
-        'nsfw_on'
-      elsif params[:nsfw_off]
-        'nsfw_off'
-      elsif params[:delete]
-        'delete'
-      end
-    end
-
-    def set_report
-      @report = Report.find(params[:report_id])
-    end
-  end
-end
diff --git a/app/controllers/admin/reports_controller.rb b/app/controllers/admin/reports_controller.rb
index 7c831b3d4..00d200d7c 100644
--- a/app/controllers/admin/reports_controller.rb
+++ b/app/controllers/admin/reports_controller.rb
@@ -13,8 +13,10 @@ module Admin
       authorize @report, :show?
 
       @report_note  = @report.notes.new
-      @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at)
-      @form         = Form::StatusBatch.new
+      @report_notes = @report.notes.includes(:account).order(id: :desc)
+      @action_logs  = @report.history.includes(:target)
+      @form         = Admin::StatusBatchAction.new
+      @statuses     = @report.statuses.with_includes
     end
 
     def assign_to_self
diff --git a/app/controllers/admin/statuses_controller.rb b/app/controllers/admin/statuses_controller.rb
index b3fd4c424..8d039b281 100644
--- a/app/controllers/admin/statuses_controller.rb
+++ b/app/controllers/admin/statuses_controller.rb
@@ -2,71 +2,57 @@
 
 module Admin
   class StatusesController < BaseController
-    helper_method :current_params
-
     before_action :set_account
+    before_action :set_statuses
 
     PER_PAGE = 20
 
     def index
       authorize :status, :index?
 
-      @statuses = @account.statuses.where(visibility: [:public, :unlisted])
-
-      if params[:media]
-        @statuses = @statuses.merge(Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)).reorder('statuses.id desc')
-      end
-
-      @statuses = @statuses.preload(:media_attachments, :mentions).page(params[:page]).per(PER_PAGE)
-      @form     = Form::StatusBatch.new
-    end
-
-    def show
-      authorize :status, :index?
-
-      @statuses = @account.statuses.where(id: params[:id])
-      authorize @statuses.first, :show?
-
-      @form = Form::StatusBatch.new
+      @status_batch_action = Admin::StatusBatchAction.new
     end
 
-    def create
-      authorize :status, :update?
-
-      @form         = Form::StatusBatch.new(form_status_batch_params.merge(current_account: current_account, action: action_from_button))
-      flash[:alert] = I18n.t('admin.statuses.failed_to_execute') unless @form.save
-
-      redirect_to admin_account_statuses_path(@account.id, current_params)
+    def batch
+      @status_batch_action = Admin::StatusBatchAction.new(admin_status_batch_action_params.merge(current_account: current_account, report_id: params[:report_id], type: action_from_button))
+      @status_batch_action.save!
     rescue ActionController::ParameterMissing
       flash[:alert] = I18n.t('admin.statuses.no_status_selected')
-
-      redirect_to admin_account_statuses_path(@account.id, current_params)
+    ensure
+      redirect_to after_create_redirect_path
     end
 
     private
 
-    def form_status_batch_params
-      params.require(:form_status_batch).permit(:action, status_ids: [])
+    def admin_status_batch_action_params
+      params.require(:admin_status_batch_action).permit(status_ids: [])
+    end
+
+    def after_create_redirect_path
+      if @status_batch_action.report_id.present?
+        admin_report_path(@status_batch_action.report_id)
+      else
+        admin_account_statuses_path(params[:account_id], current_params)
+      end
     end
 
     def set_account
       @account = Account.find(params[:account_id])
     end
 
-    def current_params
-      page = (params[:page] || 1).to_i
+    def set_statuses
+      @statuses = Admin::StatusFilter.new(@account, filter_params).results.preload(:application, :preloadable_poll, :media_attachments, active_mentions: :account, reblog: [:account, :application, :preloadable_poll, :media_attachments, active_mentions: :account]).page(params[:page]).per(PER_PAGE)
+    end
 
-      {
-        media: params[:media],
-        page: page > 1 && page,
-      }.select { |_, value| value.present? }
+    def filter_params
+      params.slice(*Admin::StatusFilter::KEYS).permit(*Admin::StatusFilter::KEYS)
     end
 
     def action_from_button
-      if params[:nsfw_on]
-        'nsfw_on'
-      elsif params[:nsfw_off]
-        'nsfw_off'
+      if params[:report]
+        'report'
+      elsif params[:remove_from_report]
+        'remove_from_report'
       elsif params[:delete]
         'delete'
       end
diff --git a/app/controllers/api/v1/admin/account_actions_controller.rb b/app/controllers/api/v1/admin/account_actions_controller.rb
index 29c9b7107..15af50822 100644
--- a/app/controllers/api/v1/admin/account_actions_controller.rb
+++ b/app/controllers/api/v1/admin/account_actions_controller.rb
@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::AccountActionsController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:accounts' }
+  protect_from_forgery with: :exception
+
+  before_action -> { authorize_if_got_token! :'admin:write', :'admin:write:accounts' }
   before_action :require_staff!
   before_action :set_account
 
diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb
index 9b8f2fb05..65330b8c8 100644
--- a/app/controllers/api/v1/admin/accounts_controller.rb
+++ b/app/controllers/api/v1/admin/accounts_controller.rb
@@ -1,13 +1,15 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::AccountsController < Api::BaseController
+  protect_from_forgery with: :exception
+
   include Authorization
   include AccountableConcern
 
   LIMIT = 100
 
-  before_action -> { doorkeeper_authorize! :'admin:read', :'admin:read:accounts' }, only: [:index, :show]
-  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:accounts' }, except: [:index, :show]
+  before_action -> { authorize_if_got_token! :'admin:read', :'admin:read:accounts' }, only: [:index, :show]
+  before_action -> { authorize_if_got_token! :'admin:write', :'admin:write:accounts' }, except: [:index, :show]
   before_action :require_staff!
   before_action :set_accounts, only: :index
   before_action :set_account, except: :index
diff --git a/app/controllers/api/v1/admin/dimensions_controller.rb b/app/controllers/api/v1/admin/dimensions_controller.rb
index 5e8f0f89f..b1f738990 100644
--- a/app/controllers/api/v1/admin/dimensions_controller.rb
+++ b/app/controllers/api/v1/admin/dimensions_controller.rb
@@ -3,6 +3,7 @@
 class Api::V1::Admin::DimensionsController < Api::BaseController
   protect_from_forgery with: :exception
 
+  before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_dimensions
 
diff --git a/app/controllers/api/v1/admin/measures_controller.rb b/app/controllers/api/v1/admin/measures_controller.rb
index f28191753..d64c3cdf7 100644
--- a/app/controllers/api/v1/admin/measures_controller.rb
+++ b/app/controllers/api/v1/admin/measures_controller.rb
@@ -3,6 +3,7 @@
 class Api::V1::Admin::MeasuresController < Api::BaseController
   protect_from_forgery with: :exception
 
+  before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_measures
 
diff --git a/app/controllers/api/v1/admin/reports_controller.rb b/app/controllers/api/v1/admin/reports_controller.rb
index c8f4cd8d8..fbfd0ee12 100644
--- a/app/controllers/api/v1/admin/reports_controller.rb
+++ b/app/controllers/api/v1/admin/reports_controller.rb
@@ -1,13 +1,15 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::ReportsController < Api::BaseController
+  protect_from_forgery with: :exception
+
   include Authorization
   include AccountableConcern
 
   LIMIT = 100
 
-  before_action -> { doorkeeper_authorize! :'admin:read', :'admin:read:reports' }, only: [:index, :show]
-  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:reports' }, except: [:index, :show]
+  before_action -> { authorize_if_got_token! :'admin:read', :'admin:read:reports' }, only: [:index, :show]
+  before_action -> { authorize_if_got_token! :'admin:write', :'admin:write:reports' }, except: [:index, :show]
   before_action :require_staff!
   before_action :set_reports, only: :index
   before_action :set_report, except: :index
@@ -32,6 +34,12 @@ class Api::V1::Admin::ReportsController < Api::BaseController
     render json: @report, serializer: REST::Admin::ReportSerializer
   end
 
+  def update
+    authorize @report, :update?
+    @report.update!(report_params)
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
   def assign_to_self
     authorize @report, :update?
     @report.update!(assigned_account_id: current_account.id)
@@ -74,6 +82,10 @@ class Api::V1::Admin::ReportsController < Api::BaseController
     ReportFilter.new(filter_params).results
   end
 
+  def report_params
+    params.permit(:category, rule_ids: [])
+  end
+
   def filter_params
     params.permit(*FILTER_PARAMS)
   end
diff --git a/app/controllers/api/v1/admin/retention_controller.rb b/app/controllers/api/v1/admin/retention_controller.rb
index a8ff64f21..4af5a5c4d 100644
--- a/app/controllers/api/v1/admin/retention_controller.rb
+++ b/app/controllers/api/v1/admin/retention_controller.rb
@@ -3,6 +3,7 @@
 class Api::V1::Admin::RetentionController < Api::BaseController
   protect_from_forgery with: :exception
 
+  before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_cohorts
 
diff --git a/app/controllers/api/v1/admin/trends/tags_controller.rb b/app/controllers/api/v1/admin/trends/tags_controller.rb
index 3653d1dd1..4815af31e 100644
--- a/app/controllers/api/v1/admin/trends/tags_controller.rb
+++ b/app/controllers/api/v1/admin/trends/tags_controller.rb
@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 class Api::V1::Admin::Trends::TagsController < Api::BaseController
+  protect_from_forgery with: :exception
+
+  before_action -> { authorize_if_got_token! :'admin:read' }
   before_action :require_staff!
   before_action :set_tags
 
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 5f69f176a..907529b37 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -13,6 +13,7 @@ module Admin::FilterHelper
     RelationshipFilter::KEYS,
     AnnouncementFilter::KEYS,
     Admin::ActionLogFilter::KEYS,
+    Admin::StatusFilter::KEYS,
   ].flatten.freeze
 
   def filter_link_to(text, link_to_params, link_class_params = link_to_params)
diff --git a/app/javascript/mastodon/components/admin/ReportReasonSelector.js b/app/javascript/mastodon/components/admin/ReportReasonSelector.js
new file mode 100644
index 000000000..1f91d2517
--- /dev/null
+++ b/app/javascript/mastodon/components/admin/ReportReasonSelector.js
@@ -0,0 +1,159 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import api from 'mastodon/api';
+import { injectIntl, defineMessages } from 'react-intl';
+import classNames from 'classnames';
+
+const messages = defineMessages({
+  other: { id: 'report.categories.other', defaultMessage: 'Other' },
+  spam: { id: 'report.categories.spam', defaultMessage: 'Spam' },
+  violation: { id: 'report.categories.violation', defaultMessage: 'Content violates one or more server rules' },
+});
+
+class Category extends React.PureComponent {
+
+  static propTypes = {
+    id: PropTypes.string.isRequired,
+    text: PropTypes.string.isRequired,
+    selected: PropTypes.bool,
+    disabled: PropTypes.bool,
+    onSelect: PropTypes.func,
+    children: PropTypes.node,
+  };
+
+  handleClick = () => {
+    const { id, disabled, onSelect } = this.props;
+
+    if (!disabled) {
+      onSelect(id);
+    }
+  };
+
+  render () {
+    const { id, text, disabled, selected, children } = this.props;
+
+    return (
+      <div tabIndex='0' role='button' className={classNames('report-reason-selector__category', { selected, disabled })} onClick={this.handleClick}>
+        {selected && <input type='hidden' name='report[category]' value={id} />}
+
+        <div className='report-reason-selector__category__label'>
+          <span className={classNames('poll__input', { active: selected, disabled })} />
+          {text}
+        </div>
+
+        {(selected && children) && (
+          <div className='report-reason-selector__category__rules'>
+            {children}
+          </div>
+        )}
+      </div>
+    );
+  }
+
+}
+
+class Rule extends React.PureComponent {
+
+  static propTypes = {
+    id: PropTypes.string.isRequired,
+    text: PropTypes.string.isRequired,
+    selected: PropTypes.bool,
+    disabled: PropTypes.bool,
+    onToggle: PropTypes.func,
+  };
+
+  handleClick = () => {
+    const { id, disabled, onToggle } = this.props;
+
+    if (!disabled) {
+      onToggle(id);
+    }
+  };
+
+  render () {
+    const { id, text, disabled, selected } = this.props;
+
+    return (
+      <div tabIndex='0' role='button' className={classNames('report-reason-selector__rule', { selected, disabled })} onClick={this.handleClick}>
+        <span className={classNames('poll__input', { checkbox: true, active: selected, disabled })} />
+        {selected && <input type='hidden' name='report[rule_ids][]' value={id} />}
+        {text}
+      </div>
+    );
+  }
+
+}
+
+export default @injectIntl
+class ReportReasonSelector extends React.PureComponent {
+
+  static propTypes = {
+    id: PropTypes.string.isRequired,
+    category: PropTypes.string.isRequired,
+    rule_ids: PropTypes.arrayOf(PropTypes.string),
+    disabled: PropTypes.bool,
+    intl: PropTypes.object.isRequired,
+  };
+
+  state = {
+    category: this.props.category,
+    rule_ids: this.props.rule_ids || [],
+    rules: [],
+  };
+
+  componentDidMount() {
+    api().get('/api/v1/instance').then(res => {
+      this.setState({
+        rules: res.data.rules,
+      });
+    }).catch(err => {
+      console.error(err);
+    });
+  }
+
+  _save = () => {
+    const { id, disabled } = this.props;
+    const { category, rule_ids } = this.state;
+
+    if (disabled) {
+      return;
+    }
+
+    api().put(`/api/v1/admin/reports/${id}`, {
+      category,
+      rule_ids,
+    }).catch(err => {
+      console.error(err);
+    });
+  };
+
+  handleSelect = id => {
+    this.setState({ category: id }, () => this._save());
+  };
+
+  handleToggle = id => {
+    const { rule_ids } = this.state;
+
+    if (rule_ids.includes(id)) {
+      this.setState({ rule_ids: rule_ids.filter(x => x !== id ) }, () => this._save());
+    } else {
+      this.setState({ rule_ids: [...rule_ids, id] }, () => this._save());
+    }
+  };
+
+  render () {
+    const { disabled, intl } = this.props;
+    const { rules, category, rule_ids } = this.state;
+
+    return (
+      <div className='report-reason-selector'>
+        <Category id='other' text={intl.formatMessage(messages.other)} selected={category === 'other'} onSelect={this.handleSelect} disabled={disabled} />
+        <Category id='spam' text={intl.formatMessage(messages.spam)} selected={category === 'spam'} onSelect={this.handleSelect} disabled={disabled} />
+        <Category id='violation' text={intl.formatMessage(messages.violation)} selected={category === 'violation'} onSelect={this.handleSelect} disabled={disabled}>
+          {rules.map(rule => <Rule key={rule.id} id={rule.id} text={rule.text} selected={rule_ids.includes(rule.id)} onToggle={this.handleToggle} disabled={disabled} />)}
+        </Category>
+      </div>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js
index d125359e9..4e19cc0e4 100644
--- a/app/javascript/mastodon/components/status_action_bar.js
+++ b/app/javascript/mastodon/components/status_action_bar.js
@@ -291,7 +291,7 @@ class StatusActionBar extends ImmutablePureComponent {
       if (isStaff) {
         menu.push(null);
         menu.push({ text: intl.formatMessage(messages.admin_account, { name: account.get('username') }), href: `/admin/accounts/${status.getIn(['account', 'id'])}` });
-        menu.push({ text: intl.formatMessage(messages.admin_status), href: `/admin/accounts/${status.getIn(['account', 'id'])}/statuses/${status.get('id')}` });
+        menu.push({ text: intl.formatMessage(messages.admin_status), href: `/admin/accounts/${status.getIn(['account', 'id'])}/statuses?id=${status.get('id')}` });
       }
     }
 
diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js
index e60119bc4..a15a4d567 100644
--- a/app/javascript/mastodon/features/status/components/action_bar.js
+++ b/app/javascript/mastodon/features/status/components/action_bar.js
@@ -245,7 +245,7 @@ class ActionBar extends React.PureComponent {
       if (isStaff) {
         menu.push(null);
         menu.push({ text: intl.formatMessage(messages.admin_account, { name: status.getIn(['account', 'username']) }), href: `/admin/accounts/${status.getIn(['account', 'id'])}` });
-        menu.push({ text: intl.formatMessage(messages.admin_status), href: `/admin/accounts/${status.getIn(['account', 'id'])}/statuses/${status.get('id')}` });
+        menu.push({ text: intl.formatMessage(messages.admin_status), href: `/admin/accounts/${status.getIn(['account', 'id'])}/statuses?id=${status.get('id')}` });
       }
     }
 
diff --git a/app/javascript/styles/mailer.scss b/app/javascript/styles/mailer.scss
index 92c02e847..34852178e 100644
--- a/app/javascript/styles/mailer.scss
+++ b/app/javascript/styles/mailer.scss
@@ -533,6 +533,10 @@ ul {
   }
 }
 
+ul.rules-list {
+  padding-top: 0;
+}
+
 @media only screen and (min-device-width: 768px) and (max-device-width: 1024px) and (orientation: landscape) {
   body {
     min-height: 1024px !important;
diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss
index dbf8a6e7a..c20762fba 100644
--- a/app/javascript/styles/mastodon/admin.scss
+++ b/app/javascript/styles/mastodon/admin.scss
@@ -579,39 +579,44 @@ body,
 
 .log-entry {
   line-height: 20px;
-  padding: 15px 0;
+  padding: 15px;
+  padding-left: 15px * 2 + 40px;
   background: $ui-base-color;
-  border-bottom: 1px solid lighten($ui-base-color, 4%);
+  border-bottom: 1px solid darken($ui-base-color, 8%);
+  position: relative;
+
+  &:first-child {
+    border-top-left-radius: 4px;
+    border-top-right-radius: 4px;
+  }
 
   &:last-child {
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
     border-bottom: 0;
   }
 
+  &:hover {
+    background: lighten($ui-base-color, 4%);
+  }
+
   &__header {
-    display: flex;
-    justify-content: flex-start;
-    align-items: center;
     color: $darker-text-color;
     font-size: 14px;
-    padding: 0 10px;
   }
 
   &__avatar {
-    margin-right: 10px;
+    position: absolute;
+    left: 15px;
+    top: 15px;
 
     .avatar {
-      display: block;
-      margin: 0;
-      border-radius: 50%;
+      border-radius: 4px;
       width: 40px;
       height: 40px;
     }
   }
 
-  &__content {
-    max-width: calc(100% - 90px);
-  }
-
   &__title {
     word-wrap: break-word;
   }
@@ -627,6 +632,14 @@ body,
     text-decoration: none;
     font-weight: 500;
   }
+
+  a {
+    &:hover,
+    &:focus,
+    &:active {
+      text-decoration: underline;
+    }
+  }
 }
 
 a.name-tag,
@@ -655,8 +668,9 @@ a.inline-name-tag,
 
 a.name-tag,
 .name-tag {
-  display: flex;
+  display: inline-flex;
   align-items: center;
+  vertical-align: top;
 
   .avatar {
     display: block;
@@ -1114,3 +1128,287 @@ a.sparkline {
     }
   }
 }
+
+.report-reason-selector {
+  border-radius: 4px;
+  background: $ui-base-color;
+  margin-bottom: 20px;
+
+  &__category {
+    cursor: pointer;
+    border-bottom: 1px solid darken($ui-base-color, 8%);
+
+    &:last-child {
+      border-bottom: 0;
+    }
+
+    &__label {
+      padding: 15px;
+    }
+
+    &__rules {
+      margin-left: 30px;
+    }
+  }
+
+  &__rule {
+    cursor: pointer;
+    padding: 15px;
+  }
+}
+
+.report-header {
+  display: grid;
+  grid-gap: 15px;
+  grid-template-columns: minmax(0, 1fr) 300px;
+
+  &__details {
+    &__item {
+      border-bottom: 1px solid lighten($ui-base-color, 8%);
+      padding: 15px 0;
+
+      &:last-child {
+        border-bottom: 0;
+      }
+
+      &__header {
+        font-weight: 600;
+        padding: 4px 0;
+      }
+    }
+
+    &--horizontal {
+      display: grid;
+      grid-auto-columns: minmax(0, 1fr);
+      grid-auto-flow: column;
+
+      .report-header__details__item {
+        border-bottom: 0;
+      }
+    }
+  }
+}
+
+.account-card {
+  background: $ui-base-color;
+  border-radius: 4px;
+
+  &__header {
+    padding: 4px;
+    border-radius: 4px;
+    height: 128px;
+
+    img {
+      display: block;
+      margin: 0;
+      width: 100%;
+      height: 100%;
+      object-fit: cover;
+      background: darken($ui-base-color, 8%);
+    }
+  }
+
+  &__title {
+    margin-top: -25px;
+    display: flex;
+    align-items: flex-end;
+
+    &__avatar {
+      padding: 15px;
+
+      img {
+        display: block;
+        margin: 0;
+        width: 56px;
+        height: 56px;
+        background: darken($ui-base-color, 8%);
+        border-radius: 8px;
+      }
+    }
+
+    .display-name {
+      color: $darker-text-color;
+      padding-bottom: 15px;
+      font-size: 15px;
+
+      bdi {
+        display: block;
+        color: $primary-text-color;
+        font-weight: 500;
+      }
+    }
+  }
+
+  &__bio {
+    padding: 0 15px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    word-wrap: break-word;
+    max-height: 18px * 2;
+    position: relative;
+
+    &::after {
+      display: block;
+      content: "";
+      width: 50px;
+      height: 18px;
+      position: absolute;
+      bottom: 0;
+      right: 15px;
+      background: linear-gradient(to left, $ui-base-color, transparent);
+      pointer-events: none;
+    }
+  }
+
+  &__actions {
+    display: flex;
+    align-items: center;
+    padding-top: 10px;
+
+    &__button {
+      flex: 0 0 auto;
+      padding: 0 15px;
+    }
+  }
+
+  &__counters {
+    flex: 1 1 auto;
+    display: grid;
+    grid-auto-columns: minmax(0, 1fr);
+    grid-auto-flow: column;
+
+    &__item {
+      padding: 15px;
+      text-align: center;
+      color: $primary-text-color;
+      font-weight: 600;
+      font-size: 15px;
+
+      small {
+        display: block;
+        color: $darker-text-color;
+        font-weight: 400;
+        font-size: 13px;
+      }
+    }
+  }
+}
+
+.report-notes {
+  margin-bottom: 20px;
+
+  &__item {
+    background: $ui-base-color;
+    position: relative;
+    padding: 15px;
+    padding-left: 15px * 2 + 40px;
+    border-bottom: 1px solid darken($ui-base-color, 8%);
+
+    &:first-child {
+      border-top-left-radius: 4px;
+      border-top-right-radius: 4px;
+    }
+
+    &:last-child {
+      border-bottom-left-radius: 4px;
+      border-bottom-right-radius: 4px;
+      border-bottom: 0;
+    }
+
+    &:hover {
+      background-color: lighten($ui-base-color, 4%);
+    }
+
+    &__avatar {
+      position: absolute;
+      left: 15px;
+      top: 15px;
+      border-radius: 4px;
+      width: 40px;
+      height: 40px;
+    }
+
+    &__header {
+      color: $darker-text-color;
+      font-size: 15px;
+      line-height: 20px;
+      margin-bottom: 4px;
+
+      .username a {
+        color: $primary-text-color;
+        font-weight: 500;
+        text-decoration: none;
+        margin-right: 5px;
+
+        &:hover,
+        &:focus,
+        &:active {
+          text-decoration: underline;
+        }
+      }
+
+      time {
+        margin-left: 5px;
+        vertical-align: baseline;
+      }
+    }
+
+    &__content {
+      font-size: 15px;
+      line-height: 20px;
+      word-wrap: break-word;
+      font-weight: 400;
+      color: $primary-text-color;
+
+      p {
+        margin-bottom: 20px;
+        white-space: pre-wrap;
+        unicode-bidi: plaintext;
+
+        &:last-child {
+          margin-bottom: 0;
+        }
+      }
+    }
+
+    &__actions {
+      position: absolute;
+      top: 15px;
+      right: 15px;
+      text-align: right;
+    }
+  }
+}
+
+.report-actions {
+  border: 1px solid darken($ui-base-color, 8%);
+
+  &__item {
+    display: flex;
+    align-items: center;
+    line-height: 18px;
+    border-bottom: 1px solid darken($ui-base-color, 8%);
+
+    &:last-child {
+      border-bottom: 0;
+    }
+
+    &__button {
+      flex: 0 0 auto;
+      width: 100px;
+      padding: 15px;
+      padding-right: 0;
+
+      .button {
+        display: block;
+        width: 100%;
+      }
+    }
+
+    &__description {
+      padding: 15px;
+      font-size: 14px;
+      color: $dark-text-color;
+    }
+  }
+}
diff --git a/app/javascript/styles/mastodon/polls.scss b/app/javascript/styles/mastodon/polls.scss
index ad7088982..e33fc7983 100644
--- a/app/javascript/styles/mastodon/polls.scss
+++ b/app/javascript/styles/mastodon/polls.scss
@@ -143,6 +143,21 @@
     &:active {
       outline: 0 !important;
     }
+
+    &.disabled {
+      border-color: $dark-text-color;
+
+      &.active {
+        background: $dark-text-color;
+      }
+
+      &:active,
+      &:focus,
+      &:hover {
+        border-color: $dark-text-color;
+        border-width: 1px;
+      }
+    }
   }
 
   &__number {
diff --git a/app/lib/admin/metrics/measure/resolved_reports_measure.rb b/app/lib/admin/metrics/measure/resolved_reports_measure.rb
index 0dcecbbad..00cb24f7e 100644
--- a/app/lib/admin/metrics/measure/resolved_reports_measure.rb
+++ b/app/lib/admin/metrics/measure/resolved_reports_measure.rb
@@ -6,11 +6,11 @@ class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure:
   end
 
   def total
-    Report.resolved.where(updated_at: time_period).count
+    Report.resolved.where(action_taken_at: time_period).count
   end
 
   def previous_total
-    Report.resolved.where(updated_at: previous_time_period).count
+    Report.resolved.where(action_taken_at: previous_time_period).count
   end
 
   def data
@@ -19,8 +19,7 @@ class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure:
         WITH resolved_reports AS (
           SELECT reports.id
           FROM reports
-          WHERE action_taken
-            AND date_trunc('day', reports.updated_at)::date = axis.period
+          WHERE date_trunc('day', reports.action_taken_at)::date = axis.period
         )
         SELECT count(*) FROM resolved_reports
       ) AS value
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index 68d1c4507..5221a4892 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -160,11 +160,11 @@ class UserMailer < Devise::Mailer
     end
   end
 
-  def warning(user, warning, status_ids = nil)
+  def warning(user, warning)
     @resource = user
     @warning  = warning
     @instance = Rails.configuration.x.local_domain
-    @statuses = Status.where(id: status_ids).includes(:account) if status_ids.is_a?(Array)
+    @statuses = @warning.statuses.includes(:account, :preloadable_poll, :media_attachments, active_mentions: [:account])
 
     I18n.with_locale(@resource.locale || I18n.default_locale) do
       mail to: @resource.email,
diff --git a/app/models/account_warning.rb b/app/models/account_warning.rb
index 5efc924d5..fc0d988fd 100644
--- a/app/models/account_warning.rb
+++ b/app/models/account_warning.rb
@@ -10,14 +10,30 @@
 #  text              :text             default(""), not null
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
+#  report_id         :bigint(8)
+#  status_ids        :string           is an Array
 #
 
 class AccountWarning < ApplicationRecord
-  enum action: %i(none disable sensitive silence suspend), _suffix: :action
+  enum action: {
+    none:            0,
+    disable:         1_000,
+    delete_statuses: 1_500,
+    sensitive:       2_000,
+    silence:         3_000,
+    suspend:         4_000,
+  }, _suffix: :action
 
   belongs_to :account, inverse_of: :account_warnings
-  belongs_to :target_account, class_name: 'Account', inverse_of: :targeted_account_warnings
+  belongs_to :target_account, class_name: 'Account', inverse_of: :strikes
+  belongs_to :report, optional: true
 
-  scope :latest, -> { order(created_at: :desc) }
+  has_one :appeal, dependent: :destroy
+
+  scope :latest, -> { order(id: :desc) }
   scope :custom, -> { where.not(text: '') }
+
+  def statuses
+    Status.with_discarded.where(id: status_ids || [])
+  end
 end
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index bf222391f..d3be4be3f 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -33,7 +33,7 @@ class Admin::AccountAction
   def save!
     ApplicationRecord.transaction do
       process_action!
-      process_warning!
+      process_strike!
     end
 
     process_email!
@@ -74,20 +74,14 @@ class Admin::AccountAction
     end
   end
 
-  def process_warning!
-    return unless warnable?
-
-    authorize(target_account, :warn?)
-
-    @warning = AccountWarning.create!(target_account: target_account,
-                                      account: current_account,
-                                      action: type,
-                                      text: text_for_warning)
-
-    # A log entry is only interesting if the warning contains
-    # custom text from someone. Otherwise it's just noise.
-
-    log_action(:create, warning) if warning.text.present?
+  def process_strike!
+    @warning = target_account.strikes.create!(
+      account: current_account,
+      report: report,
+      action: type,
+      text: text_for_warning,
+      status_ids: status_ids
+    )
   end
 
   def process_reports!
@@ -143,7 +137,7 @@ class Admin::AccountAction
   end
 
   def process_email!
-    UserMailer.warning(target_account.user, warning, status_ids).deliver_later! if warnable?
+    UserMailer.warning(target_account.user, warning).deliver_later! if warnable?
   end
 
   def warnable?
@@ -151,7 +145,7 @@ class Admin::AccountAction
   end
 
   def status_ids
-    report.status_ids if report && include_statuses
+    report.status_ids if with_report? && include_statuses
   end
 
   def reports
diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb
new file mode 100644
index 000000000..319deff98
--- /dev/null
+++ b/app/models/admin/status_batch_action.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+class Admin::StatusBatchAction
+  include ActiveModel::Model
+  include AccountableConcern
+  include Authorization
+
+  attr_accessor :current_account, :type,
+                :status_ids, :report_id
+
+  def save!
+    process_action!
+  end
+
+  private
+
+  def statuses
+    Status.with_discarded.where(id: status_ids)
+  end
+
+  def process_action!
+    return if status_ids.empty?
+
+    case type
+    when 'delete'
+      handle_delete!
+    when 'report'
+      handle_report!
+    when 'remove_from_report'
+      handle_remove_from_report!
+    end
+  end
+
+  def handle_delete!
+    statuses.each { |status| authorize(status, :destroy?) }
+
+    ApplicationRecord.transaction do
+      statuses.each do |status|
+        status.discard
+        log_action(:destroy, status)
+      end
+
+      if with_report?
+        report.resolve!(current_account)
+        log_action(:resolve, report)
+      end
+
+      @warning = target_account.strikes.create!(
+        action: :delete_statuses,
+        account: current_account,
+        report: report,
+        status_ids: status_ids
+      )
+
+      statuses.each { |status| Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) } unless target_account.local?
+    end
+
+    UserMailer.warning(target_account.user, @warning).deliver_later! if target_account.local?
+    RemovalWorker.push_bulk(status_ids) { |status_id| [status_id, preserve: target_account.local?, immediate: !target_account.local?] }
+  end
+
+  def handle_report!
+    @report = Report.new(report_params) unless with_report?
+    @report.status_ids = (@report.status_ids + status_ids.map(&:to_i)).uniq
+    @report.save!
+
+    @report_id = @report.id
+  end
+
+  def handle_remove_from_report!
+    return unless with_report?
+
+    report.status_ids -= status_ids.map(&:to_i)
+    report.save!
+  end
+
+  def report
+    @report ||= Report.find(report_id) if report_id.present?
+  end
+
+  def with_report?
+    !report.nil?
+  end
+
+  def target_account
+    @target_account ||= statuses.first.account
+  end
+
+  def report_params
+    { account: current_account, target_account: target_account }
+  end
+end
diff --git a/app/models/admin/status_filter.rb b/app/models/admin/status_filter.rb
new file mode 100644
index 000000000..ce5bb5f46
--- /dev/null
+++ b/app/models/admin/status_filter.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::StatusFilter
+  KEYS = %i(
+    media
+    id
+    report_id
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(account, params)
+    @account = account
+    @params  = params
+  end
+
+  def results
+    scope = @account.statuses.where(visibility: [:public, :unlisted])
+
+    params.each do |key, value|
+      next if %w(page report_id).include?(key.to_s)
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'media'
+      Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)
+    when 'id'
+      Status.where(id: value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index f9e7a3bea..bbe269e8f 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -42,7 +42,7 @@ module AccountAssociations
     has_many :account_moderation_notes, dependent: :destroy, inverse_of: :account
     has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account
     has_many :account_warnings, dependent: :destroy, inverse_of: :account
-    has_many :targeted_account_warnings, class_name: 'AccountWarning', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account
+    has_many :strikes, class_name: 'AccountWarning', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account
 
     # Lists (that the account is on, not owned by the account)
     has_many :list_accounts, inverse_of: :account, dependent: :destroy
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
deleted file mode 100644
index c4943a7ea..000000000
--- a/app/models/form/status_batch.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-class Form::StatusBatch
-  include ActiveModel::Model
-  include AccountableConcern
-
-  attr_accessor :status_ids, :action, :current_account
-
-  def save
-    case action
-    when 'nsfw_on', 'nsfw_off'
-      change_sensitive(action == 'nsfw_on')
-    when 'delete'
-      delete_statuses
-    end
-  end
-
-  private
-
-  def change_sensitive(sensitive)
-    media_attached_status_ids = MediaAttachment.where(status_id: status_ids).pluck(:status_id)
-
-    ApplicationRecord.transaction do
-      Status.where(id: media_attached_status_ids).reorder(nil).find_each do |status|
-        status.update!(sensitive: sensitive)
-        log_action :update, status
-      end
-    end
-
-    true
-  rescue ActiveRecord::RecordInvalid
-    false
-  end
-
-  def delete_statuses
-    Status.where(id: status_ids).reorder(nil).find_each do |status|
-      status.discard
-      RemovalWorker.perform_async(status.id, immediate: true)
-      Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
-      log_action :destroy, status
-    end
-
-    true
-  end
-end
diff --git a/app/models/report.rb b/app/models/report.rb
index ef41547d9..ceb15133b 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -6,7 +6,6 @@
 #  id                         :bigint(8)        not null, primary key
 #  status_ids                 :bigint(8)        default([]), not null, is an Array
 #  comment                    :text             default(""), not null
-#  action_taken               :boolean          default(FALSE), not null
 #  created_at                 :datetime         not null
 #  updated_at                 :datetime         not null
 #  account_id                 :bigint(8)        not null
@@ -15,9 +14,14 @@
 #  assigned_account_id        :bigint(8)
 #  uri                        :string
 #  forwarded                  :boolean
+#  category                   :integer          default("other"), not null
+#  action_taken_at            :datetime
+#  rule_ids                   :bigint(8)        is an Array
 #
 
 class Report < ApplicationRecord
+  self.ignored_columns = %w(action_taken)
+
   include Paginable
   include RateLimitable
 
@@ -30,11 +34,17 @@ class Report < ApplicationRecord
 
   has_many :notes, class_name: 'ReportNote', foreign_key: :report_id, inverse_of: :report, dependent: :destroy
 
-  scope :unresolved, -> { where(action_taken: false) }
-  scope :resolved,   -> { where(action_taken: true) }
+  scope :unresolved, -> { where(action_taken_at: nil) }
+  scope :resolved,   -> { where.not(action_taken_at: nil) }
   scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) }
 
-  validates :comment, length: { maximum: 1000 }
+  validates :comment, length: { maximum: 1_000 }
+
+  enum category: {
+    other: 0,
+    spam: 1_000,
+    violation: 2_000,
+  }
 
   def local?
     false # Force uri_for to use uri attribute
@@ -47,13 +57,17 @@ class Report < ApplicationRecord
   end
 
   def statuses
-    Status.with_discarded.where(id: status_ids).includes(:account, :media_attachments, :mentions)
+    Status.with_discarded.where(id: status_ids)
   end
 
   def media_attachments
     MediaAttachment.where(status_id: status_ids)
   end
 
+  def rules
+    Rule.with_discarded.where(id: rule_ids)
+  end
+
   def assign_to_self!(current_account)
     update!(assigned_account_id: current_account.id)
   end
@@ -63,22 +77,19 @@ class Report < ApplicationRecord
   end
 
   def resolve!(acting_account)
-    if account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
-      # This is an automated report and it is being dismissed, so it's
-      # a false positive, in which case update the account's trust level
-      # to prevent further spam checks
-
-      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
-    end
-
-    RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] }
-    update!(action_taken: true, action_taken_by_account_id: acting_account.id)
+    update!(action_taken_at: Time.now.utc, action_taken_by_account_id: acting_account.id)
   end
 
   def unresolve!
-    update!(action_taken: false, action_taken_by_account_id: nil)
+    update!(action_taken_at: nil, action_taken_by_account_id: nil)
+  end
+
+  def action_taken?
+    action_taken_at.present?
   end
 
+  alias action_taken action_taken?
+
   def unresolved?
     !action_taken?
   end
@@ -88,29 +99,24 @@ class Report < ApplicationRecord
   end
 
   def history
-    time_range = created_at..updated_at
-
-    sql = [
+    subquery = [
       Admin::ActionLog.where(
         target_type: 'Report',
-        target_id: id,
-        created_at: time_range
-      ).unscope(:order),
+        target_id: id
+      ).unscope(:order).arel,
 
       Admin::ActionLog.where(
         target_type: 'Account',
-        target_id: target_account_id,
-        created_at: time_range
-      ).unscope(:order),
+        target_id: target_account_id
+      ).unscope(:order).arel,
 
       Admin::ActionLog.where(
         target_type: 'Status',
-        target_id: status_ids,
-        created_at: time_range
-      ).unscope(:order),
-    ].map { |query| "(#{query.to_sql})" }.join(' UNION ALL ')
+        target_id: status_ids
+      ).unscope(:order).arel,
+    ].reduce { |union, query| Arel::Nodes::UnionAll.new(union, query) }
 
-    Admin::ActionLog.from("(#{sql}) AS admin_action_logs")
+    Admin::ActionLog.from(Arel::Nodes::As.new(subquery, Admin::ActionLog.arel_table))
   end
 
   def set_uri
diff --git a/app/models/report_filter.rb b/app/models/report_filter.rb
index a91a6baeb..dc444a552 100644
--- a/app/models/report_filter.rb
+++ b/app/models/report_filter.rb
@@ -19,7 +19,7 @@ class ReportFilter
     scope = Report.unresolved
 
     params.each do |key, value|
-      scope = scope.merge scope_for(key, value)
+      scope = scope.merge scope_for(key, value), rewhere: true
     end
 
     scope
diff --git a/app/serializers/rest/admin/report_serializer.rb b/app/serializers/rest/admin/report_serializer.rb
index 7a77132c0..74bc0c520 100644
--- a/app/serializers/rest/admin/report_serializer.rb
+++ b/app/serializers/rest/admin/report_serializer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class REST::Admin::ReportSerializer < ActiveModel::Serializer
-  attributes :id, :action_taken, :comment, :created_at, :updated_at
+  attributes :id, :action_taken, :category, :comment, :created_at, :updated_at
 
   has_one :account, serializer: REST::Admin::AccountSerializer
   has_one :target_account, serializer: REST::Admin::AccountSerializer
@@ -9,8 +9,13 @@ class REST::Admin::ReportSerializer < ActiveModel::Serializer
   has_one :action_taken_by_account, serializer: REST::Admin::AccountSerializer
 
   has_many :statuses, serializer: REST::StatusSerializer
+  has_many :rules, serializer: REST::RuleSerializer
 
   def id
     object.id.to_s
   end
+
+  def statuses
+    object.statuses.with_includes
+  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index f9c3dcf78..3535b503b 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -9,6 +9,7 @@ class RemoveStatusService < BaseService
   # @param   [Hash] options
   # @option  [Boolean] :redraft
   # @option  [Boolean] :immediate
+  # @option  [Boolean] :preserve
   # @option  [Boolean] :original_removed
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
@@ -43,7 +44,7 @@ class RemoveStatusService < BaseService
           remove_media
         end
 
-        @status.destroy! if @options[:immediate] || !@status.reported?
+        @status.destroy! if permanently?
       else
         raise Mastodon::RaceConditionError
       end
@@ -135,11 +136,15 @@ class RemoveStatusService < BaseService
   end
 
   def remove_media
-    return if @options[:redraft] || (!@options[:immediate] && @status.reported?)
+    return if @options[:redraft] || !permanently?
 
     @status.media_attachments.destroy_all
   end
 
+  def permanently?
+    @options[:immediate] || !(@options[:preserve] || @status.reported?)
+  end
+
   def lock_options
     { redis: Redis.current, key: "distribute:#{@status.id}", autorelease: 5.minutes.seconds }
   end
diff --git a/app/views/admin/action_logs/index.html.haml b/app/views/admin/action_logs/index.html.haml
index f7f73150b..f611bfe9d 100644
--- a/app/views/admin/action_logs/index.html.haml
+++ b/app/views/admin/action_logs/index.html.haml
@@ -22,7 +22,7 @@
   %div.muted-hint.center-text
     = t 'admin.action_logs.empty'
 - else
-  .announcements-list
+  .report-notes
     = render partial: 'action_log', collection: @action_logs
 
 = paginate @action_logs
diff --git a/app/views/admin/report_notes/_report_note.html.haml b/app/views/admin/report_notes/_report_note.html.haml
index d34dc3d15..428b6cf59 100644
--- a/app/views/admin/report_notes/_report_note.html.haml
+++ b/app/views/admin/report_notes/_report_note.html.haml
@@ -1,7 +1,18 @@
-.speech-bubble
-  .speech-bubble__bubble
+.report-notes__item
+  = image_tag report_note.account.avatar.url, class: 'report-notes__item__avatar'
+
+  .report-notes__item__header
+    %span.username
+      = link_to display_name(report_note.account), admin_account_path(report_note.account_id)
+    %time{ datetime: report_note.created_at.iso8601, title: l(report_note.created_at) }
+      - if report_note.created_at.today?
+        = t('admin.report_notes.today_at', time: l(report_note.created_at, format: :time))
+      - else
+        = l report_note.created_at.to_date
+
+  .report-notes__item__content
     = simple_format(h(report_note.content))
-  .speech-bubble__owner
-    = admin_account_link_to report_note.account
-    %time.formatted{ datetime: report_note.created_at.iso8601 }= l report_note.created_at
-    = table_link_to 'trash', t('admin.reports.notes.delete'), admin_report_note_path(report_note), method: :delete if can?(:destroy, report_note)
+
+  - if can?(:destroy, report_note)
+    .report-notes__item__actions
+      = table_link_to 'trash', t('admin.reports.notes.delete'), admin_report_note_path(report_note), method: :delete
diff --git a/app/views/admin/reports/_action_log.html.haml b/app/views/admin/reports/_action_log.html.haml
deleted file mode 100644
index 0f7d05867..000000000
--- a/app/views/admin/reports/_action_log.html.haml
+++ /dev/null
@@ -1,6 +0,0 @@
-.speech-bubble.positive
-  .speech-bubble__bubble
-    = t("admin.action_logs.actions.#{action_log.action}_#{action_log.target_type.underscore}_html", name: content_tag(:span, action_log.account.username, class: 'username'), target: content_tag(:span, log_target(action_log), class: 'target'))
-  .speech-bubble__owner
-    = admin_account_link_to(action_log.account)
-    %time.formatted{ datetime: action_log.created_at.iso8601 }= l action_log.created_at
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index ada6dd2bc..924b0e9c2 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -22,6 +22,9 @@
         = react_component :media_gallery, height: 343, sensitive: status.proper.sensitive?, visible: false, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
+      - if status.application
+        = status.application.name
+        ·
       = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener noreferrer' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
       - if status.discarded?
diff --git a/app/views/admin/reports/show.html.haml b/app/views/admin/reports/show.html.haml
index b060c553f..4f513dd39 100644
--- a/app/views/admin/reports/show.html.haml
+++ b/app/views/admin/reports/show.html.haml
@@ -1,5 +1,6 @@
 - content_for :header_tags do
   = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
+  = javascript_pack_tag 'public', async: true, crossorigin: 'anonymous'
 
 - content_for :page_title do
   = t('admin.reports.report', id: @report.id)
@@ -10,122 +11,199 @@
   - else
     = link_to t('admin.reports.mark_as_unresolved'), reopen_admin_report_path(@report), method: :post, class: 'button'
 
-.table-wrapper
-  %table.table.inline-table
-    %tbody
-      %tr
-        %th= t('admin.reports.reported_account')
-        %td= admin_account_link_to @report.target_account
-        %td= table_link_to 'flag', t('admin.reports.account.reports', count: @report.target_account.targeted_reports.count), admin_reports_path(target_account_id: @report.target_account.id)
-        %td= table_link_to 'file', t('admin.reports.account.notes', count: @report.target_account.targeted_moderation_notes.count), admin_reports_path(target_account_id: @report.target_account.id)
-      %tr
-        %th= t('admin.reports.reported_by')
+.report-header
+  .report-header__card
+    .account-card
+      .account-card__header
+        = image_tag @report.target_account.header.url, alt: ''
+      .account-card__title
+        .account-card__title__avatar
+          = image_tag @report.target_account.avatar.url, alt: ''
+        .display-name
+          %bdi
+            %strong.emojify.p-name= display_name(@report.target_account, custom_emojify: true)
+          %span
+            = acct(@report.target_account)
+            = fa_icon('lock') if @report.target_account.locked?
+      - if @report.target_account.note.present?
+        .account-card__bio.emojify
+          = Formatter.instance.simplified_format(@report.target_account, custom_emojify: true)
+      .account-card__actions
+        .account-card__counters
+          .account-card__counters__item
+            = friendly_number_to_human @report.target_account.statuses_count
+            %small= t('accounts.posts', count: @report.target_account.statuses_count).downcase
+          .account-card__counters__item
+            = friendly_number_to_human @report.target_account.followers_count
+            %small= t('accounts.followers', count: @report.target_account.followers_count).downcase
+          .account-card__counters__item
+            = friendly_number_to_human @report.target_account.following_count
+            %small= t('accounts.following', count: @report.target_account.following_count).downcase
+        .account-card__actions__button
+          = link_to t('admin.reports.view_profile'), admin_account_path(@report.target_account_id), class: 'button'
+    .report-header__details.report-header__details--horizontal
+      .report-header__details__item
+        .report-header__details__item__header
+          %strong= t('admin.accounts.joined')
+        .report-header__details__item__content
+          %time.time-ago{ datetime: @report.target_account.created_at.iso8601, title: l(@report.target_account.created_at) }= l @report.target_account.created_at
+      .report-header__details__item
+        .report-header__details__item__header
+          %strong= t('accounts.last_active')
+        .report-header__details__item__content
+          - if @report.target_account.last_status_at.present?
+            %time.time-ago{ datetime: @report.target_account.last_status_at.to_date.iso8601, title: l(@report.target_account.last_status_at.to_date) }= l @report.target_account.last_status_at
+      .report-header__details__item
+        .report-header__details__item__header
+          %strong= t('admin.accounts.strikes')
+        .report-header__details__item__content
+          = @report.target_account.strikes.count
+
+  .report-header__details
+    .report-header__details__item
+      .report-header__details__item__header
+        %strong= t('admin.reports.created_at')
+      .report-header__details__item__content
+        %time.formatted{ datetime: @report.created_at.iso8601 }
+    .report-header__details__item
+      .report-header__details__item__header
+        %strong= t('admin.reports.reported_by')
+      .report-header__details__item__content
         - if @report.account.instance_actor?
-          %td{ colspan: 3 }= site_hostname
+          = site_hostname
         - elsif @report.account.local?
-          %td= admin_account_link_to @report.account
-          %td= table_link_to 'flag', t('admin.reports.account.reports', count: @report.account.targeted_reports.count), admin_reports_path(target_account_id: @report.account.id)
-          %td= table_link_to 'file', t('admin.reports.account.notes', count: @report.account.targeted_moderation_notes.count), admin_reports_path(target_account_id: @report.account.id)
+          = admin_account_link_to @report.account
+        - else
+          = @report.account.domain
+    .report-header__details__item
+      .report-header__details__item__header
+        %strong= t('admin.reports.status')
+      .report-header__details__item__content
+        - if @report.action_taken?
+          = t('admin.reports.resolved')
         - else
-          %td{ colspan: 3 }= @report.account.domain
-      %tr
-        %th= t('admin.reports.created_at')
-        %td{ colspan: 3 }
-          %time.formatted{ datetime: @report.created_at.iso8601 }
-      %tr
-        %th= t('admin.reports.updated_at')
-        %td{ colspan: 3 }
-          %time.formatted{ datetime: @report.updated_at.iso8601 }
-      %tr
-        %th= t('admin.reports.status')
-        %td
-          - if @report.action_taken?
-            = t('admin.reports.resolved')
+          = t('admin.reports.unresolved')
+    - unless @report.target_account.local?
+      .report-header__details__item
+        .report-header__details__item__header
+          %strong= t('admin.reports.forwarded')
+        .report-header__details__item__content
+          - if @report.forwarded?
+            = t('simple_form.yes')
           - else
-            = t('admin.reports.unresolved')
-        %td{ colspan: 2 }
-          - if @report.action_taken?
-            = table_link_to 'envelope-open', t('admin.reports.reopen'), admin_report_path(@report, outcome: 'reopen'), method: :put
-      - unless @report.target_account.local?
-        %tr
-          %th= t('admin.reports.forwarded')
-          %td{ colspan: 3 }
-            - if @report.forwarded.nil?
-              \-
-            - elsif @report.forwarded?
-              = t('simple_form.yes')
-            - else
-              = t('simple_form.no')
-      - if !@report.action_taken_by_account.nil?
-        %tr
-          %th= t('admin.reports.action_taken_by')
-          %td{ colspan: 3 }
-            = admin_account_link_to @report.action_taken_by_account
-      - else
-        %tr
-          %th= t('admin.reports.assigned')
-          %td
-            - if @report.assigned_account.nil?
-              \-
-            - else
-              = admin_account_link_to @report.assigned_account
-          %td
-            - if @report.assigned_account != current_user.account
-              = table_link_to 'user', t('admin.reports.assign_to_self'), assign_to_self_admin_report_path(@report), method: :post
-          %td
-            - if !@report.assigned_account.nil?
-              = table_link_to 'trash', t('admin.reports.unassign'), unassign_admin_report_path(@report), method: :post
+            = t('simple_form.no')
+    - if !@report.action_taken_by_account.nil?
+      .report-header__details__item
+        .report-header__details__item__header
+          %strong= t('admin.reports.action_taken_by')
+        .report-header__details__item__content
+          = admin_account_link_to @report.action_taken_by_account
+    - else
+      .report-header__details__item
+        .report-header__details__item__header
+          %strong= t('admin.reports.assigned')
+        .report-header__details__item__content
+          - if @report.assigned_account.nil?
+            = t 'admin.reports.no_one_assigned'
+          - else
+            = admin_account_link_to @report.assigned_account
+          —
+          - if @report.assigned_account != current_user.account
+            = table_link_to 'user', t('admin.reports.assign_to_self'), assign_to_self_admin_report_path(@report), method: :post
+          - elsif !@report.assigned_account.nil?
+            = table_link_to 'trash', t('admin.reports.unassign'), unassign_admin_report_path(@report), method: :post
 
 %hr.spacer
 
-%div.action-buttons
-  %div
+%h3= t 'admin.reports.category'
 
-  - if @report.unresolved?
-    %div
-      - if @report.target_account.local?
-        = link_to t('admin.accounts.warn'), new_admin_account_action_path(@report.target_account_id, type: 'none', report_id: @report.id), class: 'button'
-        = link_to t('admin.accounts.disable'), new_admin_account_action_path(@report.target_account_id, type: 'disable', report_id: @report.id), class: 'button button--destructive'
-      = link_to t('admin.accounts.silence'), new_admin_account_action_path(@report.target_account_id, type: 'silence', report_id: @report.id), class: 'button button--destructive'
-      = link_to t('admin.accounts.perform_full_suspension'), new_admin_account_action_path(@report.target_account_id, type: 'suspend', report_id: @report.id), class: 'button button--destructive'
+%p= t 'admin.reports.category_description_html'
 
-%hr.spacer
+= react_admin_component :report_reason_selector, id: @report.id, category: @report.category, rule_ids: @report.rule_ids&.map(&:to_s), disabled: @report.action_taken?
 
-.speech-bubble
-  .speech-bubble__bubble= simple_format(@report.comment.presence || t('admin.reports.comment.none'))
-  .speech-bubble__owner
-    - if @report.account.local?
-      = admin_account_link_to @report.account
-    - else
-      = @report.account.domain
-      %br/
-    %time.formatted{ datetime: @report.created_at.iso8601 }
+- if @report.comment.present?
+  %p= t('admin.reports.comment_description_html', name: content_tag(:strong, @report.account.username, class: 'username'))
+
+  .report-notes__item
+    = image_tag @report.account.avatar.url, class: 'report-notes__item__avatar'
+
+    .report-notes__item__header
+      %span.username
+        = link_to display_name(@report.account), admin_account_path(@report.account_id)
+      %time{ datetime: @report.created_at.iso8601, title: l(@report.created_at) }
+        - if @report.created_at.today?
+          = t('admin.report_notes.today_at', time: l(@report.created_at, format: :time))
+        - else
+          = l @report.created_at.to_date
+
+    .report-notes__item__content
+      = simple_format(h(@report.comment))
+
+%hr.spacer/
 
-- unless @report.statuses.empty?
+%h3= t 'admin.reports.statuses'
+
+%p
+  = t 'admin.reports.statuses_description_html'
+  —
+  = link_to safe_join([fa_icon('plus'), t('admin.reports.add_to_report')]), admin_account_statuses_path(@report.target_account_id, report_id: @report.id), class: 'table-action-link'
+
+= form_for(@form, url: batch_admin_account_statuses_path(@report.target_account_id, report_id: @report.id)) do |f|
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        - if !@statuses.empty? && @report.unresolved?
+          = f.button safe_join([fa_icon('times'), t('admin.statuses.batch.remove_from_report')]), name: :remove_from_report, class: 'table-action-link', type: :submit
+          = f.button safe_join([fa_icon('trash'), t('admin.reports.delete_and_resolve')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        - else
+    .batch-table__body
+      - if @statuses.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'admin/reports/status', collection: @statuses, locals: { f: f }
+
+- if @report.unresolved?
   %hr.spacer/
 
-  = form_for(@form, url: admin_report_reported_statuses_path(@report.id)) do |f|
-    .batch-table
-      .batch-table__toolbar
-        %label.batch-table__toolbar__select.batch-checkbox-all
-          = check_box_tag :batch_checkbox_all, nil, false
-        .batch-table__toolbar__actions
-          = f.button safe_join([fa_icon('eye-slash'), t('admin.statuses.batch.nsfw_on')]), name: :nsfw_on, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-          = f.button safe_join([fa_icon('eye'), t('admin.statuses.batch.nsfw_off')]), name: :nsfw_off, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-          = f.button safe_join([fa_icon('trash'), t('admin.statuses.batch.delete')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-      .batch-table__body
-        = render partial: 'admin/reports/status', collection: @report.statuses, locals: { f: f }
+  %p= t 'admin.reports.actions_description_html'
+
+  .report-actions
+    .report-actions__item
+      .report-actions__item__button
+        = link_to t('admin.accounts.silence'), new_admin_account_action_path(@report.target_account_id, type: 'silence', report_id: @report.id), class: 'button button--destructive'
+      .report-actions__item__description
+        = t('admin.reports.actions.silence_description_html')
+    .report-actions__item
+      .report-actions__item__button
+        = link_to t('admin.accounts.perform_full_suspension'), new_admin_account_action_path(@report.target_account_id, report_id: @report.id, type: 'suspend'), class: 'button button--destructive'
+      .report-actions__item__description
+        = t('admin.reports.actions.suspend_description_html')
+    .report-actions__item
+      .report-actions__item__button
+        = link_to t('admin.accounts.custom'), new_admin_account_action_path(@report.target_account_id, report_id: @report.id), class: 'button'
+      .report-actions__item__description
+        = t('admin.reports.actions.other_description_html')
+
+- unless @action_logs.empty?
+  %hr.spacer/
+
+  %h3= t 'admin.reports.action_log'
+
+  .report-notes
+    = render @action_logs
 
 %hr.spacer/
 
-- @report_notes.each do |item|
-  - if item.is_a?(Admin::ActionLog)
-    = render partial: 'action_log', locals: { action_log: item }
-  - else
-    = render item
+%h3= t 'admin.reports.notes.title'
+
+%p= t 'admin.reports.notes_description_html'
+
+.report-notes
+  = render @report_notes
 
 = simple_form_for @report_note, url: admin_report_notes_path do |f|
-  = render 'shared/error_messages', object: @report_note
   = f.input :report_id, as: :hidden
 
   .field-group
diff --git a/app/views/admin/statuses/index.html.haml b/app/views/admin/statuses/index.html.haml
index c39ba9071..7e2114cc2 100644
--- a/app/views/admin/statuses/index.html.haml
+++ b/app/views/admin/statuses/index.html.haml
@@ -10,28 +10,37 @@
   .filter-subset
     %strong= t('admin.statuses.media.title')
     %ul
-      %li= link_to t('admin.statuses.no_media'), admin_account_statuses_path(@account.id, current_params.merge(media: nil)), class: !params[:media] && 'selected'
-      %li= link_to t('admin.statuses.with_media'), admin_account_statuses_path(@account.id, current_params.merge(media: true)), class: params[:media] && 'selected'
+      %li= filter_link_to t('generic.all'), media: nil, id: nil
+      %li= filter_link_to t('admin.statuses.with_media'), media: '1'
   .back-link
-    = link_to admin_account_path(@account.id) do
-      = fa_icon 'chevron-left fw'
-      = t('admin.statuses.back_to_account')
+    - if params[:report_id]
+      = link_to admin_report_path(params[:report_id].to_i) do
+        = fa_icon 'chevron-left fw'
+        = t('admin.statuses.back_to_report')
+    - else
+      = link_to admin_account_path(@account.id) do
+        = fa_icon 'chevron-left fw'
+        = t('admin.statuses.back_to_account')
 
 %hr.spacer/
 
-= form_for(@form, url: admin_account_statuses_path(@account.id)) do |f|
-  = hidden_field_tag :page, params[:page]
-  = hidden_field_tag :media, params[:media]
+= form_for(@status_batch_action, url: batch_admin_account_statuses_path(@account.id)) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - Admin::StatusFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
 
   .batch-table
     .batch-table__toolbar
       %label.batch-table__toolbar__select.batch-checkbox-all
         = check_box_tag :batch_checkbox_all, nil, false
       .batch-table__toolbar__actions
-        = f.button safe_join([fa_icon('eye-slash'), t('admin.statuses.batch.nsfw_on')]), name: :nsfw_on, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-        = f.button safe_join([fa_icon('eye'), t('admin.statuses.batch.nsfw_off')]), name: :nsfw_off, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-        = f.button safe_join([fa_icon('trash'), t('admin.statuses.batch.delete')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        - unless @statuses.empty?
+          = f.button safe_join([fa_icon('flag'), t('admin.statuses.batch.report')]), name: :report, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
     .batch-table__body
-      = render partial: 'admin/reports/status', collection: @statuses, locals: { f: f }
+      - if @statuses.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'admin/reports/status', collection: @statuses, locals: { f: f }
 
 = paginate @statuses
diff --git a/app/views/admin/statuses/show.html.haml b/app/views/admin/statuses/show.html.haml
deleted file mode 100644
index e2470198d..000000000
--- a/app/views/admin/statuses/show.html.haml
+++ /dev/null
@@ -1,27 +0,0 @@
-- content_for :page_title do
-  = t('admin.statuses.title')
-  \-
-  = "@#{@account.acct}"
-
-.filters
-  .back-link
-    = link_to admin_account_path(@account.id) do
-      %i.fa.fa-chevron-left.fa-fw
-      = t('admin.statuses.back_to_account')
-
-%hr.spacer/
-
-= form_for(@form, url: admin_account_statuses_path(@account.id)) do |f|
-  = hidden_field_tag :page, params[:page]
-  = hidden_field_tag :media, params[:media]
-
-  .batch-table
-    .batch-table__toolbar
-      %label.batch-table__toolbar__select.batch-checkbox-all
-        = check_box_tag :batch_checkbox_all, nil, false
-      .batch-table__toolbar__actions
-        = f.button safe_join([fa_icon('eye-slash'), t('admin.statuses.batch.nsfw_on')]), name: :nsfw_on, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-        = f.button safe_join([fa_icon('eye'), t('admin.statuses.batch.nsfw_off')]), name: :nsfw_off, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-        = f.button safe_join([fa_icon('trash'), t('admin.statuses.batch.delete')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-    .batch-table__body
-      = render partial: 'admin/reports/status', collection: @statuses, locals: { f: f }
diff --git a/app/views/notification_mailer/_status.text.erb b/app/views/notification_mailer/_status.text.erb
index 8999a1f8e..c43f32d9f 100644
--- a/app/views/notification_mailer/_status.text.erb
+++ b/app/views/notification_mailer/_status.text.erb
@@ -1,8 +1,8 @@
 <% if status.spoiler_text? %>
-<%= raw status.spoiler_text %>
-----
-
+> <%= raw word_wrap(status.spoiler_text, break_sequence: "\n> ") %>
+> ----
+>
 <% end %>
-<%= raw Formatter.instance.plaintext(status) %>
+> <%= raw word_wrap(Formatter.instance.plaintext(status), break_sequence: "\n> ") %>
 
 <%= raw t('application_mailer.view')%> <%= web_url("statuses/#{status.id}") %>
diff --git a/app/views/user_mailer/warning.html.haml b/app/views/user_mailer/warning.html.haml
index 5a2911ecb..bda1fef6c 100644
--- a/app/views/user_mailer/warning.html.haml
+++ b/app/views/user_mailer/warning.html.haml
@@ -37,16 +37,26 @@
                           %tr
                             %td.column-cell.text-center
                               - unless @warning.none_action?
-                                %p= t "user_mailer.warning.explanation.#{@warning.action}"
+                                %p= t "user_mailer.warning.explanation.#{@warning.action}", instance: @instance
 
                               - unless @warning.text.blank?
                                 = Formatter.instance.linkify(@warning.text)
 
-                              - if !@statuses.nil? && !@statuses.empty?
+                              - if @warning.report && !@warning.report.other?
+                                %p
+                                  %strong= t('user_mailer.warning.reason')
+                                  = t("user_mailer.warning.categories.#{@warning.report.category}")
+
+                                - if @warning.report.violation? && @warning.report.rule_ids.present?
+                                  %ul.rules-list
+                                    - @warning.report.rules.each do |rule|
+                                      %li= rule.text
+
+                              - unless @statuses.empty?
                                 %p
                                   %strong= t('user_mailer.warning.statuses')
 
-- if !@statuses.nil? && !@statuses.empty?
+- unless @statuses.empty?
   - @statuses.each_with_index do |status, i|
     = render 'notification_mailer/status', status: status, i: i + 1, highlighted: true
 
diff --git a/app/views/user_mailer/warning.text.erb b/app/views/user_mailer/warning.text.erb
index bb6610c79..31d7308ae 100644
--- a/app/views/user_mailer/warning.text.erb
+++ b/app/views/user_mailer/warning.text.erb
@@ -3,11 +3,24 @@
 ===
 
 <% unless @warning.none_action? %>
-<%= t "user_mailer.warning.explanation.#{@warning.action}" %>
+<%= t "user_mailer.warning.explanation.#{@warning.action}", instance: @instance %>
 
 <% end %>
+<% if @warning.text.present? %>
 <%= @warning.text %>
-<% if !@statuses.nil? && !@statuses.empty? %>
+
+<% end %>
+<% if @warning.report && !@warning.report.other? %>
+**<%= t('user_mailer.warning.reason') %>** <%= t("user_mailer.warning.categories.#{@warning.report.category}") %>
+
+<% if @warning.report.violation? && @warning.report.rule_ids.present? %>
+<% @warning.report.rules.each do |rule| %>
+- <%= rule.text %>
+<% end %>
+
+<% end %>
+<% end %>
+<% if !@statuses.empty? %>
 <%= t('user_mailer.warning.statuses') %>
 
 <% @statuses.each do |status| %>
diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb
index be0c4277d..d06b637f9 100644
--- a/app/workers/scheduler/user_cleanup_scheduler.rb
+++ b/app/workers/scheduler/user_cleanup_scheduler.rb
@@ -8,6 +8,7 @@ class Scheduler::UserCleanupScheduler
   def perform
     clean_unconfirmed_accounts!
     clean_suspended_accounts!
+    clean_discarded_statuses!
   end
 
   private
@@ -24,4 +25,12 @@ class Scheduler::UserCleanupScheduler
       Admin::AccountDeletionWorker.perform_async(deletion_request.account_id)
     end
   end
+
+  def clean_discarded_statuses!
+    Status.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses|
+      RemovalWorker.push_bulk(statuses) do |status|
+        [status.id, { immediate: true }]
+      end
+    end
+  end
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 693a7b400..36ac89664 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -113,6 +113,7 @@ en:
       confirm: Confirm
       confirmed: Confirmed
       confirming: Confirming
+      custom: Custom
       delete: Delete data
       deleted: Deleted
       demote: Demote
@@ -203,6 +204,7 @@ en:
       silence: Limit
       silenced: Limited
       statuses: Posts
+      strikes: Previous strikes
       subscribe: Subscribe
       suspended: Suspended
       suspension_irreversible: The data of this account has been irreversibly deleted. You can unsuspend the account to make it usable but it will not recover any data it previously had.
@@ -549,32 +551,44 @@ en:
     report_notes:
       created_msg: Report note successfully created!
       destroyed_msg: Report note successfully deleted!
+      today_at: Today at %{time}
     reports:
       account:
         notes:
           one: "%{count} note"
           other: "%{count} notes"
-        reports:
-          one: "%{count} report"
-          other: "%{count} reports"
+      action_log: Audit log
       action_taken_by: Action taken by
+      actions:
+        other_description_html: See more options for controlling the account's behaviour and customize communication to the reported account.
+        silence_description_html: The profile will be visible only to those who already follow it or manually look it up, severely limiting its reach. Can always be reverted.
+        suspend_description_html: The profile and all its contents will become inaccessible until it is eventually deleted. Interacting with the account will be impossible. Reversible within 30 days.
+      actions_description_html: 'If removing the offending content above is insufficient:'
+      add_to_report: Add more to report
       are_you_sure: Are you sure?
       assign_to_self: Assign to me
       assigned: Assigned moderator
       by_target_domain: Domain of reported account
+      category: Category
+      category_description_html: The reason this account and/or content was reported will be cited in communication with the reported account
       comment:
         none: None
+      comment_description_html: 'To provide more information, %{name} wrote:'
       created_at: Reported
+      delete_and_resolve: Delete and resolve
       forwarded: Forwarded
       forwarded_to: Forwarded to %{domain}
       mark_as_resolved: Mark as resolved
       mark_as_unresolved: Mark as unresolved
+      no_one_assigned: No one
       notes:
         create: Add note
         create_and_resolve: Resolve with note
         create_and_unresolve: Reopen with note
         delete: Delete
         placeholder: Describe what actions have been taken, or any other related updates...
+        title: Notes
+      notes_description_html: View and leave notes to other moderators and your future self
       reopen: Reopen report
       report: 'Report #%{id}'
       reported_account: Reported account
@@ -582,11 +596,14 @@ en:
       resolved: Resolved
       resolved_msg: Report successfully resolved!
       status: Status
+      statuses: Reported content
+      statuses_description_html: Offending content will be cited in communication with the reported account
       target_origin: Origin of reported account
       title: Reports
       unassign: Unassign
       unresolved: Unresolved
       updated_at: Updated
+      view_profile: View profile
     rules:
       add_new: Add rule
       delete: Delete
@@ -688,15 +705,13 @@ en:
       destroyed_msg: Site upload successfully deleted!
     statuses:
       back_to_account: Back to account page
+      back_to_report: Back to report page
       batch:
-        delete: Delete
-        nsfw_off: Mark as not sensitive
-        nsfw_on: Mark as sensitive
+        remove_from_report: Remove from report
+        report: Report
       deleted: Deleted
-      failed_to_execute: Failed to execute
       media:
         title: Media
-      no_media: No media
       no_status_selected: No posts were changed as none were selected
       title: Account posts
       with_media: With media
@@ -1457,6 +1472,7 @@ en:
     formats:
       default: "%b %d, %Y, %H:%M"
       month: "%b %Y"
+      time: "%H:%M"
   two_factor_authentication:
     add: Add
     disable: Disable 2FA
@@ -1484,24 +1500,31 @@ en:
       subject: Please confirm attempted sign in
       title: Sign in attempt
     warning:
+      categories:
+        spam: Spam
+        violation: Content violates the following community guidelines
       explanation:
-        disable: You can no longer login to your account or use it in any other way, but your profile and other data remains intact.
-        sensitive: Your uploaded media files and linked media will be treated as sensitive.
-        silence: You can still use your account but only people who are already following you will see your posts on this server, and you may be excluded from various public listings. However, others may still manually follow you.
-        suspend: You can no longer use your account, and your profile and other data are no longer accessible. You can still login to request a backup of your data until the data is fully removed, but we will retain some data to prevent you from evading the suspension.
-      get_in_touch: You can reply to this e-mail to get in touch with the staff of %{instance}.
+        delete_statuses: Some of your posts have been found to violate one or more community guidelines and have been subsequently removed by the moderators of %{instance}. Future violations may result in harsher punitive actions against your account.
+        disable: You can no longer use your account, but your profile and other data remains intact. You can request a backup of your data, change account settings or delete your account.
+        sensitive: From now on, all your uploaded media files will be marked as sensitive and hidden behind a click-through warning.
+        silence: You can still use your account but only people who are already following you will see your posts on this server, and you may be excluded from various discovery features. However, others may still manually follow you.
+        suspend: You can no longer use your account, and your profile and other data are no longer accessible. You can still login to request a backup of your data until the data is fully removed in about 30 days, but we will retain some basic data to prevent you from evading the suspension.
+      get_in_touch: If you believe this is an error, you can reply to this e-mail to get in touch with the staff of %{instance}.
+      reason: 'Reason:'
       review_server_policies: Review server policies
-      statuses: 'Specifically, for:'
+      statuses: 'Posts that have been found in violation:'
       subject:
+        delete_statuses: Your posts on %{acct} have been removed
         disable: Your account %{acct} has been frozen
         none: Warning for %{acct}
-        sensitive: Your account %{acct} posting media has been marked as sensitive
+        sensitive: Your media files on %{acct} will be marked as sensitive from now on
         silence: Your account %{acct} has been limited
         suspend: Your account %{acct} has been suspended
       title:
+        delete_statuses: Posts removed
         disable: Account frozen
         none: Warning
-        sensitive: Your media has been marked as sensitive
+        sensitive: Media hidden
         silence: Account limited
         suspend: Account suspended
     welcome:
diff --git a/config/routes.rb b/config/routes.rb
index 2357ab6c7..41ba45379 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -231,8 +231,6 @@ Rails.application.routes.draw do
         post :reopen
         post :resolve
       end
-
-      resources :reported_statuses, only: [:create]
     end
 
     resources :report_notes, only: [:create, :destroy]
@@ -259,7 +257,13 @@ Rails.application.routes.draw do
       resource :change_email, only: [:show, :update]
       resource :reset, only: [:create]
       resource :action, only: [:new, :create], controller: 'account_actions'
-      resources :statuses, only: [:index, :show, :create, :update, :destroy]
+
+      resources :statuses, only: [:index] do
+        collection do
+          post :batch
+        end
+      end
+
       resources :relationships, only: [:index]
 
       resource :confirmation, only: [:create] do
@@ -514,7 +518,7 @@ Rails.application.routes.draw do
           resource :action, only: [:create], controller: 'account_actions'
         end
 
-        resources :reports, only: [:index, :show] do
+        resources :reports, only: [:index, :update, :show] do
           member do
             post :assign_to_self
             post :unassign
diff --git a/db/migrate/20211231080958_add_category_to_reports.rb b/db/migrate/20211231080958_add_category_to_reports.rb
new file mode 100644
index 000000000..c2b495c63
--- /dev/null
+++ b/db/migrate/20211231080958_add_category_to_reports.rb
@@ -0,0 +1,21 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddCategoryToReports < ActiveRecord::Migration[6.1]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :reports, :category, :int, default: 0, allow_null: false }
+    add_column :reports, :action_taken_at, :datetime
+    add_column :reports, :rule_ids, :bigint, array: true
+    safety_assured { execute 'UPDATE reports SET action_taken_at = updated_at WHERE action_taken = TRUE' }
+  end
+
+  def down
+    safety_assured { execute 'UPDATE reports SET action_taken = TRUE WHERE action_taken_at IS NOT NULL' }
+    remove_column :reports, :category
+    remove_column :reports, :action_taken_at
+    remove_column :reports, :rule_ids
+  end
+end
diff --git a/db/migrate/20220115125126_add_report_id_to_account_warnings.rb b/db/migrate/20220115125126_add_report_id_to_account_warnings.rb
new file mode 100644
index 000000000..a1c20c99e
--- /dev/null
+++ b/db/migrate/20220115125126_add_report_id_to_account_warnings.rb
@@ -0,0 +1,6 @@
+class AddReportIdToAccountWarnings < ActiveRecord::Migration[6.1]
+  def change
+    safety_assured { add_reference :account_warnings, :report, foreign_key: { on_delete: :cascade }, index: false }
+    add_column :account_warnings, :status_ids, :string, array: true
+  end
+end
diff --git a/db/migrate/20220115125341_fix_account_warning_actions.rb b/db/migrate/20220115125341_fix_account_warning_actions.rb
new file mode 100644
index 000000000..25cc17fd3
--- /dev/null
+++ b/db/migrate/20220115125341_fix_account_warning_actions.rb
@@ -0,0 +1,21 @@
+class FixAccountWarningActions < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      execute 'UPDATE account_warnings SET action = 1000 WHERE action = 1'
+      execute 'UPDATE account_warnings SET action = 2000 WHERE action = 2'
+      execute 'UPDATE account_warnings SET action = 3000 WHERE action = 3'
+      execute 'UPDATE account_warnings SET action = 4000 WHERE action = 4'
+    end
+  end
+
+  def down
+    safety_assured do
+      execute 'UPDATE account_warnings SET action = 1 WHERE action = 1000'
+      execute 'UPDATE account_warnings SET action = 2 WHERE action = 2000'
+      execute 'UPDATE account_warnings SET action = 3 WHERE action = 3000'
+      execute 'UPDATE account_warnings SET action = 4 WHERE action = 4000'
+    end
+  end
+end
diff --git a/db/migrate/20220116202951_add_deleted_at_index_on_statuses.rb b/db/migrate/20220116202951_add_deleted_at_index_on_statuses.rb
new file mode 100644
index 000000000..dc3362552
--- /dev/null
+++ b/db/migrate/20220116202951_add_deleted_at_index_on_statuses.rb
@@ -0,0 +1,7 @@
+class AddDeletedAtIndexOnStatuses < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :statuses, :deleted_at, where: 'deleted_at IS NOT NULL', algorithm: :concurrently
+  end
+end
diff --git a/db/post_migrate/20220109213908_remove_action_taken_from_reports.rb b/db/post_migrate/20220109213908_remove_action_taken_from_reports.rb
new file mode 100644
index 000000000..73e6ad6f4
--- /dev/null
+++ b/db/post_migrate/20220109213908_remove_action_taken_from_reports.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class RemoveActionTakenFromReports < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    safety_assured { remove_column :reports, :action_taken, :boolean, default: false, null: false }
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index d1446c652..ed615a1ee 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_12_13_040746) do
+ActiveRecord::Schema.define(version: 2022_01_16_202951) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -133,6 +133,8 @@ ActiveRecord::Schema.define(version: 2021_12_13_040746) do
     t.text "text", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.bigint "report_id"
+    t.string "status_ids", array: true
     t.index ["account_id"], name: "index_account_warnings_on_account_id"
     t.index ["target_account_id"], name: "index_account_warnings_on_target_account_id"
   end
@@ -747,7 +749,6 @@ ActiveRecord::Schema.define(version: 2021_12_13_040746) do
   create_table "reports", force: :cascade do |t|
     t.bigint "status_ids", default: [], null: false, array: true
     t.text "comment", default: "", null: false
-    t.boolean "action_taken", default: false, null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.bigint "account_id", null: false
@@ -756,6 +757,9 @@ ActiveRecord::Schema.define(version: 2021_12_13_040746) do
     t.bigint "assigned_account_id"
     t.string "uri"
     t.boolean "forwarded"
+    t.integer "category", default: 0, null: false
+    t.datetime "action_taken_at"
+    t.bigint "rule_ids", array: true
     t.index ["account_id"], name: "index_reports_on_account_id"
     t.index ["target_account_id"], name: "index_reports_on_target_account_id"
   end
@@ -851,6 +855,7 @@ ActiveRecord::Schema.define(version: 2021_12_13_040746) do
     t.bigint "poll_id"
     t.datetime "deleted_at"
     t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
+    t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
     t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
     t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
@@ -1008,6 +1013,7 @@ ActiveRecord::Schema.define(version: 2021_12_13_040746) do
   add_foreign_key "account_statuses_cleanup_policies", "accounts", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", on_delete: :nullify
+  add_foreign_key "account_warnings", "reports", on_delete: :cascade
   add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify
   add_foreign_key "admin_action_logs", "accounts", on_delete: :cascade
   add_foreign_key "announcement_mutes", "accounts", on_delete: :cascade
diff --git a/spec/controllers/admin/report_notes_controller_spec.rb b/spec/controllers/admin/report_notes_controller_spec.rb
index ec5872c7d..c0013f41a 100644
--- a/spec/controllers/admin/report_notes_controller_spec.rb
+++ b/spec/controllers/admin/report_notes_controller_spec.rb
@@ -12,11 +12,11 @@ describe Admin::ReportNotesController do
   describe 'POST #create' do
     subject { post :create, params: params }
 
-    let(:report) { Fabricate(:report, action_taken: action_taken, action_taken_by_account_id: account_id) }
+    let(:report) { Fabricate(:report, action_taken_at: action_taken, action_taken_by_account_id: account_id) }
 
     context 'when parameter is valid' do
       context 'when report is unsolved' do
-        let(:action_taken) { false }
+        let(:action_taken) { nil }
         let(:account_id) { nil }
 
         context 'when create_and_resolve flag is on' do
@@ -41,7 +41,7 @@ describe Admin::ReportNotesController do
       end
 
       context 'when report is resolved' do
-        let(:action_taken) { true }
+        let(:action_taken) { Time.now.utc }
         let(:account_id) { user.account.id }
 
         context 'when create_and_unresolve flag is on' do
@@ -68,7 +68,7 @@ describe Admin::ReportNotesController do
 
     context 'when parameter is invalid' do
       let(:params) { { report_note: { content: '', report_id: report.id } } }
-      let(:action_taken) { false }
+      let(:action_taken) { nil }
       let(:account_id) { nil }
 
       it 'renders admin/reports/show' do
diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb
deleted file mode 100644
index 2a1598123..000000000
--- a/spec/controllers/admin/reported_statuses_controller_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-require 'rails_helper'
-
-describe Admin::ReportedStatusesController do
-  render_views
-
-  let(:user) { Fabricate(:user, admin: true) }
-  let(:report) { Fabricate(:report, status_ids: [status.id]) }
-  let(:status) { Fabricate(:status) }
-
-  before do
-    sign_in user, scope: :user
-  end
-
-  describe 'POST #create' do
-    subject do
-      -> { post :create, params: { :report_id => report, action => '', :form_status_batch => { status_ids: status_ids } } }
-    end
-
-    let(:action) { 'nsfw_on' }
-    let(:status_ids) { [status.id] }
-    let(:status) { Fabricate(:status, sensitive: !sensitive) }
-    let(:sensitive) { true }
-    let!(:media_attachment) { Fabricate(:media_attachment, status: status) }
-
-    context 'when action is nsfw_on' do
-      it 'updates sensitive column' do
-        is_expected.to change {
-          status.reload.sensitive
-        }.from(false).to(true)
-      end
-    end
-
-    context 'when action is nsfw_off' do
-      let(:action) { 'nsfw_off' }
-      let(:sensitive) { false }
-
-      it 'updates sensitive column' do
-        is_expected.to change {
-          status.reload.sensitive
-        }.from(true).to(false)
-      end
-    end
-
-    context 'when action is delete' do
-      let(:action) { 'delete' }
-
-      it 'removes a status' do
-        allow(RemovalWorker).to receive(:perform_async)
-        subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true)
-      end
-    end
-
-    it 'redirects to report page' do
-      subject.call
-      expect(response).to redirect_to(admin_report_path(report))
-    end
-  end
-end
diff --git a/spec/controllers/admin/reports_controller_spec.rb b/spec/controllers/admin/reports_controller_spec.rb
index 49d3e9707..d421f0739 100644
--- a/spec/controllers/admin/reports_controller_spec.rb
+++ b/spec/controllers/admin/reports_controller_spec.rb
@@ -10,8 +10,8 @@ describe Admin::ReportsController do
 
   describe 'GET #index' do
     it 'returns http success with no filters' do
-      specified = Fabricate(:report, action_taken: false)
-      Fabricate(:report, action_taken: true)
+      specified = Fabricate(:report, action_taken_at: nil)
+      Fabricate(:report, action_taken_at: Time.now.utc)
 
       get :index
 
@@ -22,10 +22,10 @@ describe Admin::ReportsController do
     end
 
     it 'returns http success with resolved filter' do
-      specified = Fabricate(:report, action_taken: true)
-      Fabricate(:report, action_taken: false)
+      specified = Fabricate(:report, action_taken_at: Time.now.utc)
+      Fabricate(:report, action_taken_at: nil)
 
-      get :index, params: { resolved: 1 }
+      get :index, params: { resolved: '1' }
 
       reports = assigns(:reports).to_a
       expect(reports.size).to eq 1
@@ -54,15 +54,7 @@ describe Admin::ReportsController do
       expect(response).to redirect_to(admin_reports_path)
       report.reload
       expect(report.action_taken_by_account).to eq user.account
-      expect(report.action_taken).to eq true
-    end
-
-    it 'sets trust level when the report is an antispam one' do
-      report = Fabricate(:report, account: Account.representative)
-
-      put :resolve, params: { id: report }
-      report.reload
-      expect(report.target_account.trust_level).to eq Account::TRUST_LEVELS[:trusted]
+      expect(report.action_taken?).to eq true
     end
   end
 
@@ -74,7 +66,7 @@ describe Admin::ReportsController do
       expect(response).to redirect_to(admin_report_path(report))
       report.reload
       expect(report.action_taken_by_account).to eq nil
-      expect(report.action_taken).to eq false
+      expect(report.action_taken?).to eq false
     end
   end
 
diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb
index e388caae2..de32fd18e 100644
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@@ -18,65 +18,46 @@ describe Admin::StatusesController do
   end
 
   describe 'GET #index' do
-    it 'returns http success with no media' do
-      get :index, params: { account_id: account.id }
+    context do
+      before do
+        get :index, params: { account_id: account.id }
+      end
 
-      statuses = assigns(:statuses).to_a
-      expect(statuses.size).to eq 4
-      expect(statuses.first.id).to eq last_status.id
-      expect(response).to have_http_status(200)
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
     end
 
-    it 'returns http success with media' do
-      get :index, params: { account_id: account.id, media: true }
+    context 'filtering by media' do
+      before do
+        get :index, params: { account_id: account.id, media: '1' }
+      end
 
-      statuses = assigns(:statuses).to_a
-      expect(statuses.size).to eq 2
-      expect(statuses.first.id).to eq last_media_attached_status.id
-      expect(response).to have_http_status(200)
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
     end
   end
 
-  describe 'POST #create' do
-    subject do
-      -> { post :create, params: { :account_id => account.id, action => '', :form_status_batch => { status_ids: status_ids } } }
+  describe 'POST #batch' do
+    before do
+      post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } }
     end
 
-    let(:action) { 'nsfw_on' }
     let(:status_ids) { [media_attached_status.id] }
 
-    context 'when action is nsfw_on' do
-      it 'updates sensitive column' do
-        is_expected.to change {
-          media_attached_status.reload.sensitive
-        }.from(false).to(true)
-      end
-    end
+    context 'when action is report' do
+      let(:action) { 'report' }
 
-    context 'when action is nsfw_off' do
-      let(:action) { 'nsfw_off' }
-      let(:sensitive) { false }
-
-      it 'updates sensitive column' do
-        is_expected.to change {
-          media_attached_status.reload.sensitive
-        }.from(true).to(false)
+      it 'creates a report' do
+        report = Report.last
+        expect(report.target_account_id).to eq account.id
+        expect(report.status_ids).to eq status_ids
       end
-    end
-
-    context 'when action is delete' do
-      let(:action) { 'delete' }
 
-      it 'removes a status' do
-        allow(RemovalWorker).to receive(:perform_async)
-        subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true)
+      it 'redirects to report page' do
+        expect(response).to redirect_to(admin_report_path(Report.last.id))
       end
     end
-
-    it 'redirects to account statuses page' do
-      subject.call
-      expect(response).to redirect_to(admin_account_statuses_path(account.id))
-    end
   end
 end
diff --git a/spec/fabricators/report_fabricator.rb b/spec/fabricators/report_fabricator.rb
index 5bd4a63f0..2c7101e09 100644
--- a/spec/fabricators/report_fabricator.rb
+++ b/spec/fabricators/report_fabricator.rb
@@ -1,6 +1,6 @@
 Fabricator(:report) do
   account
-  target_account { Fabricate(:account) }
-  comment      "You nasty"
-  action_taken false
+  target_account  { Fabricate(:account) }
+  comment         "You nasty"
+  action_taken_at nil
 end
diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb
index 6d87fd706..69b9b971e 100644
--- a/spec/mailers/previews/user_mailer_preview.rb
+++ b/spec/mailers/previews/user_mailer_preview.rb
@@ -79,7 +79,7 @@ class UserMailerPreview < ActionMailer::Preview
 
   # Preview this email at http://localhost:3000/rails/mailers/user_mailer/warning
   def warning
-    UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence), [Status.first.id])
+    UserMailer.warning(User.first, AccountWarning.last)
   end
 
   # Preview this email at http://localhost:3000/rails/mailers/user_mailer/sign_in_token
diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb
deleted file mode 100644
index 68d84a737..000000000
--- a/spec/models/form/status_batch_spec.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-require 'rails_helper'
-
-describe Form::StatusBatch do
-  let(:form) { Form::StatusBatch.new(action: action, status_ids: status_ids) }
-  let(:status) { Fabricate(:status) }
-
-  describe 'with nsfw action' do
-    let(:status_ids) { [status.id, nonsensitive_status.id, sensitive_status.id] }
-    let(:nonsensitive_status) { Fabricate(:status, sensitive: false) }
-    let(:sensitive_status) { Fabricate(:status, sensitive: true) }
-    let!(:shown_media_attachment) { Fabricate(:media_attachment, status: nonsensitive_status) }
-    let!(:hidden_media_attachment) { Fabricate(:media_attachment, status: sensitive_status) }
-
-    context 'nsfw_on' do
-      let(:action) { 'nsfw_on' }
-
-      it { expect(form.save).to be true }
-      it { expect { form.save }.to change { nonsensitive_status.reload.sensitive }.from(false).to(true) }
-      it { expect { form.save }.not_to change { sensitive_status.reload.sensitive } }
-      it { expect { form.save }.not_to change { status.reload.sensitive } }
-    end
-
-    context 'nsfw_off' do
-      let(:action) { 'nsfw_off' }
-
-      it { expect(form.save).to be true }
-      it { expect { form.save }.to change { sensitive_status.reload.sensitive }.from(true).to(false) }
-      it { expect { form.save }.not_to change { nonsensitive_status.reload.sensitive } }
-      it { expect { form.save }.not_to change { status.reload.sensitive } }
-    end
-  end
-
-  describe 'with delete action' do
-    let(:status_ids) { [status.id] }
-    let(:action) { 'delete' }
-    let!(:another_status) { Fabricate(:status) }
-
-    before do
-      allow(RemovalWorker).to receive(:perform_async)
-    end
-
-    it 'call RemovalWorker' do
-      form.save
-      expect(RemovalWorker).to have_received(:perform_async).with(status.id, immediate: true)
-    end
-
-    it 'do not call RemovalWorker' do
-      form.save
-      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, immediate: true)
-    end
-  end
-end
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
index 312954c9d..3d29c0219 100644
--- a/spec/models/report_spec.rb
+++ b/spec/models/report_spec.rb
@@ -54,7 +54,7 @@ describe Report do
   end
 
   describe 'resolve!' do
-    subject(:report) { Fabricate(:report, action_taken: false, action_taken_by_account_id: nil) }
+    subject(:report) { Fabricate(:report, action_taken_at: nil, action_taken_by_account_id: nil) }
 
     let(:acting_account) { Fabricate(:account) }
 
@@ -63,12 +63,13 @@ describe Report do
     end
 
     it 'records action taken' do
-      expect(report).to have_attributes(action_taken: true, action_taken_by_account_id: acting_account.id)
+      expect(report.action_taken?).to be true
+      expect(report.action_taken_by_account_id).to eq acting_account.id
     end
   end
 
   describe 'unresolve!' do
-    subject(:report) { Fabricate(:report, action_taken: true, action_taken_by_account_id: acting_account.id) }
+    subject(:report) { Fabricate(:report, action_taken_at: Time.now.utc, action_taken_by_account_id: acting_account.id) }
 
     let(:acting_account) { Fabricate(:account) }
 
@@ -77,23 +78,24 @@ describe Report do
     end
 
     it 'unresolves' do
-      expect(report).to have_attributes(action_taken: false, action_taken_by_account_id: nil)
+      expect(report.action_taken?).to be false
+      expect(report.action_taken_by_account_id).to be_nil
     end
   end
 
   describe 'unresolved?' do
     subject { report.unresolved? }
 
-    let(:report) { Fabricate(:report, action_taken: action_taken) }
+    let(:report) { Fabricate(:report, action_taken_at: action_taken) }
 
     context 'if action is taken' do
-      let(:action_taken) { true }
+      let(:action_taken) { Time.now.utc }
 
       it { is_expected.to be false }
     end
 
     context 'if action not is taken' do
-      let(:action_taken) { false }
+      let(:action_taken) { nil }
 
       it { is_expected.to be true }
     end
-- 
cgit 


From 2d1f082bb6bee89242ee8042dc19016179078566 Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Wed, 19 Jan 2022 12:08:46 +0900
Subject: Fix NameError on ActivityPub::FetchFeaturedCollectionService (#17326)

Related: #16954
---
 app/services/activitypub/fetch_featured_collection_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index 9fce478c1..780741feb 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -48,6 +48,6 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
   end
 
   def local_follower
-    @local_follower ||= account.followers.local.without_suspended.first
+    @local_follower ||= @account.followers.local.without_suspended.first
   end
 end
-- 
cgit 


From 1060666c583670bb3b89ed5154e61038331e30c3 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 19 Jan 2022 22:37:27 +0100
Subject: Add support for editing for published statuses (#16697)

* Add support for editing for published statuses

* Fix references to stripped-out code

* Various fixes and improvements

* Further fixes and improvements

* Fix updates being potentially sent to unauthorized recipients

* Various fixes and improvements

* Fix wrong words in test

* Fix notifying accounts that were tagged but were not in the audience

* Fix mistake
---
 .../api/v1/statuses/histories_controller.rb        |  21 ++
 .../api/v1/statuses/sources_controller.rb          |  21 ++
 app/helpers/jsonld_helper.rb                       |   8 +-
 .../mastodon/actions/importer/normalizer.js        |   7 +-
 app/javascript/mastodon/actions/statuses.js        |   3 +
 app/javascript/mastodon/actions/streaming.js       |   4 +
 app/javascript/mastodon/components/status.js       |   3 +-
 .../features/status/components/detailed_status.js  |  14 +-
 app/javascript/styles/mastodon/components.scss     |  11 +
 app/lib/activitypub/activity.rb                    |  43 ----
 app/lib/activitypub/activity/announce.rb           |  18 +-
 app/lib/activitypub/activity/create.rb             | 249 +++++++------------
 app/lib/activitypub/activity/update.rb             |  17 +-
 app/lib/activitypub/parser/custom_emoji_parser.rb  |  27 ++
 .../activitypub/parser/media_attachment_parser.rb  |  58 +++++
 app/lib/activitypub/parser/poll_parser.rb          |  53 ++++
 app/lib/activitypub/parser/status_parser.rb        | 118 +++++++++
 app/lib/feed_manager.rb                            |  20 +-
 app/lib/status_reach_finder.rb                     |  31 ++-
 app/models/poll.rb                                 |   1 +
 app/models/status.rb                               |   7 +
 app/models/status_edit.rb                          |  23 ++
 app/serializers/activitypub/note_serializer.rb     |   7 +
 app/serializers/rest/status_edit_serializer.rb     |   6 +
 app/serializers/rest/status_serializer.rb          |   2 +-
 app/serializers/rest/status_source_serializer.rb   |   9 +
 .../activitypub/fetch_remote_poll_service.rb       |   2 +-
 app/services/activitypub/process_poll_service.rb   |  64 -----
 .../activitypub/process_status_update_service.rb   | 275 +++++++++++++++++++++
 app/services/fan_out_on_write_service.rb           | 149 ++++++-----
 app/services/process_mentions_service.rb           |  65 ++---
 app/services/remove_status_service.rb              |   2 +-
 app/workers/activitypub/distribution_worker.rb     |  48 +---
 app/workers/activitypub/raw_distribution_worker.rb |  37 ++-
 .../activitypub/reply_distribution_worker.rb       |  34 ---
 .../activitypub/update_distribution_worker.rb      |  25 +-
 app/workers/distribution_worker.rb                 |   4 +-
 app/workers/feed_insert_worker.rb                  |  34 ++-
 app/workers/local_notification_worker.rb           |   2 +
 app/workers/poll_expiration_notify_worker.rb       |  45 +++-
 app/workers/push_update_worker.rb                  |  35 ++-
 config/routes.rb                                   |   3 +
 .../20210904215403_add_edited_at_to_statuses.rb    |   5 +
 db/migrate/20210908220918_create_status_edits.rb   |  13 +
 db/schema.rb                                       |  15 ++
 .../api/v1/statuses/histories_controller_spec.rb   |  29 +++
 .../api/v1/statuses/sources_controller_spec.rb     |  29 +++
 spec/fabricators/preview_card_fabricator.rb        |   6 +
 spec/fabricators/status_edit_fabricator.rb         |   7 +
 spec/lib/status_reach_finder_spec.rb               | 109 ++++++++
 spec/models/status_edit_spec.rb                    |   5 +
 .../fetch_remote_status_service_spec.rb            |   6 +-
 spec/services/fan_out_on_write_service_spec.rb     | 107 ++++++--
 spec/services/process_mentions_service_spec.rb     |  32 +--
 .../activitypub/distribution_worker_spec.rb        |   7 +-
 spec/workers/feed_insert_worker_spec.rb            |   2 +-
 56 files changed, 1409 insertions(+), 568 deletions(-)
 create mode 100644 app/controllers/api/v1/statuses/histories_controller.rb
 create mode 100644 app/controllers/api/v1/statuses/sources_controller.rb
 create mode 100644 app/lib/activitypub/parser/custom_emoji_parser.rb
 create mode 100644 app/lib/activitypub/parser/media_attachment_parser.rb
 create mode 100644 app/lib/activitypub/parser/poll_parser.rb
 create mode 100644 app/lib/activitypub/parser/status_parser.rb
 create mode 100644 app/models/status_edit.rb
 create mode 100644 app/serializers/rest/status_edit_serializer.rb
 create mode 100644 app/serializers/rest/status_source_serializer.rb
 delete mode 100644 app/services/activitypub/process_poll_service.rb
 create mode 100644 app/services/activitypub/process_status_update_service.rb
 delete mode 100644 app/workers/activitypub/reply_distribution_worker.rb
 create mode 100644 db/migrate/20210904215403_add_edited_at_to_statuses.rb
 create mode 100644 db/migrate/20210908220918_create_status_edits.rb
 create mode 100644 spec/controllers/api/v1/statuses/histories_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/statuses/sources_controller_spec.rb
 create mode 100644 spec/fabricators/preview_card_fabricator.rb
 create mode 100644 spec/fabricators/status_edit_fabricator.rb
 create mode 100644 spec/lib/status_reach_finder_spec.rb
 create mode 100644 spec/models/status_edit_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/api/v1/statuses/histories_controller.rb b/app/controllers/api/v1/statuses/histories_controller.rb
new file mode 100644
index 000000000..c2c1fac5d
--- /dev/null
+++ b/app/controllers/api/v1/statuses/histories_controller.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::HistoriesController < Api::BaseController
+  include Authorization
+
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
+  before_action :set_status
+
+  def show
+    render json: @status.edits, each_serializer: REST::StatusEditSerializer
+  end
+
+  private
+
+  def set_status
+    @status = Status.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    not_found
+  end
+end
diff --git a/app/controllers/api/v1/statuses/sources_controller.rb b/app/controllers/api/v1/statuses/sources_controller.rb
new file mode 100644
index 000000000..434086451
--- /dev/null
+++ b/app/controllers/api/v1/statuses/sources_controller.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::SourcesController < Api::BaseController
+  include Authorization
+
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }
+  before_action :set_status
+
+  def show
+    render json: @status, serializer: REST::StatusSourceSerializer
+  end
+
+  private
+
+  def set_status
+    @status = Status.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    not_found
+  end
+end
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 62eb50f78..c24d2ddf1 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -34,7 +34,13 @@ module JsonLdHelper
   end
 
   def as_array(value)
-    value.is_a?(Array) ? value : [value]
+    if value.nil?
+      []
+    elsif value.is_a?(Array)
+      value
+    else
+      [value]
+    end
   end
 
   def value_or_id(value)
diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js
index 6b79e1f16..ca76e3494 100644
--- a/app/javascript/mastodon/actions/importer/normalizer.js
+++ b/app/javascript/mastodon/actions/importer/normalizer.js
@@ -54,9 +54,10 @@ export function normalizeStatus(status, normalOldStatus) {
     normalStatus.poll = status.poll.id;
   }
 
-  // Only calculate these values when status first encountered
-  // Otherwise keep the ones already in the reducer
-  if (normalOldStatus) {
+  // Only calculate these values when status first encountered and
+  // when the underlying values change. Otherwise keep the ones
+  // already in the reducer
+  if (normalOldStatus && normalOldStatus.get('content') === normalStatus.content && normalOldStatus.get('spoiler_text') === normalStatus.spoiler_text) {
     normalStatus.search_index = normalOldStatus.get('search_index');
     normalStatus.contentHtml = normalOldStatus.get('contentHtml');
     normalStatus.spoilerHtml = normalOldStatus.get('spoilerHtml');
diff --git a/app/javascript/mastodon/actions/statuses.js b/app/javascript/mastodon/actions/statuses.js
index 3fc7c0702..20d71362e 100644
--- a/app/javascript/mastodon/actions/statuses.js
+++ b/app/javascript/mastodon/actions/statuses.js
@@ -131,6 +131,9 @@ export function deleteStatusFail(id, error) {
   };
 };
 
+export const updateStatus = status => dispatch =>
+  dispatch(importFetchedStatus(status));
+
 export function fetchContext(id) {
   return (dispatch, getState) => {
     dispatch(fetchContextRequest(id));
diff --git a/app/javascript/mastodon/actions/streaming.js b/app/javascript/mastodon/actions/streaming.js
index beb5c6a4a..8fbb22271 100644
--- a/app/javascript/mastodon/actions/streaming.js
+++ b/app/javascript/mastodon/actions/streaming.js
@@ -10,6 +10,7 @@ import {
 } from './timelines';
 import { updateNotifications, expandNotifications } from './notifications';
 import { updateConversations } from './conversations';
+import { updateStatus } from './statuses';
 import {
   fetchAnnouncements,
   updateAnnouncements,
@@ -75,6 +76,9 @@ export const connectTimelineStream = (timelineId, channelName, params = {}, opti
         case 'update':
           dispatch(updateTimeline(timelineId, JSON.parse(data.payload), options.accept));
           break;
+        case 'status.update':
+          dispatch(updateStatus(JSON.parse(data.payload)));
+          break;
         case 'delete':
           dispatch(deleteFromTimelines(data.payload));
           break;
diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index 9955046c0..fb370ca71 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -57,6 +57,7 @@ const messages = defineMessages({
   unlisted_short: { id: 'privacy.unlisted.short', defaultMessage: 'Unlisted' },
   private_short: { id: 'privacy.private.short', defaultMessage: 'Followers-only' },
   direct_short: { id: 'privacy.direct.short', defaultMessage: 'Direct' },
+  edited: { id: 'status.edited', defaultMessage: 'Edited {date}' },
 });
 
 export default @injectIntl
@@ -483,7 +484,7 @@ class Status extends ImmutablePureComponent {
             <div className='status__info'>
               <a onClick={this.handleClick} href={status.get('url')} className='status__relative-time' target='_blank' rel='noopener noreferrer'>
                 <span className='status__visibility-icon'><Icon id={visibilityIcon.icon} title={visibilityIcon.text} /></span>
-                <RelativeTimestamp timestamp={status.get('created_at')} />
+                <RelativeTimestamp timestamp={status.get('created_at')} />{status.get('edited_at') && <abbr title={intl.formatMessage(messages.edited, { date: intl.formatDate(status.get('edited_at'), { hour12: false, year: 'numeric', month: 'short', day: '2-digit', hour: '2-digit', minute: '2-digit' }) })}> *</abbr>}
               </a>
 
               <a onClick={this.handleAccountClick} href={status.getIn(['account', 'url'])} title={status.getIn(['account', 'acct'])} className='status__display-name' target='_blank' rel='noopener noreferrer'>
diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js
index 72ddeb2b2..ee4a6b989 100644
--- a/app/javascript/mastodon/features/status/components/detailed_status.js
+++ b/app/javascript/mastodon/features/status/components/detailed_status.js
@@ -6,7 +6,7 @@ import DisplayName from '../../../components/display_name';
 import StatusContent from '../../../components/status_content';
 import MediaGallery from '../../../components/media_gallery';
 import { Link } from 'react-router-dom';
-import { injectIntl, defineMessages, FormattedDate } from 'react-intl';
+import { injectIntl, defineMessages, FormattedDate, FormattedMessage } from 'react-intl';
 import Card from './card';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import Video from '../../video';
@@ -116,6 +116,7 @@ class DetailedStatus extends ImmutablePureComponent {
     let reblogLink = '';
     let reblogIcon = 'retweet';
     let favouriteLink = '';
+    let edited = '';
 
     if (this.props.measureHeight) {
       outerStyle.height = `${this.state.height}px`;
@@ -237,6 +238,15 @@ class DetailedStatus extends ImmutablePureComponent {
       );
     }
 
+    if (status.get('edited_at')) {
+      edited = (
+        <React.Fragment>
+          <React.Fragment> · </React.Fragment>
+          <FormattedMessage id='status.edited' defaultMessage='Edited {date}' values={{ date: intl.formatDate(status.get('edited_at'), { hour12: false, month: 'short', day: '2-digit', hour: '2-digit', minute: '2-digit' }) }} />
+        </React.Fragment>
+      );
+    }
+
     return (
       <div style={outerStyle}>
         <div ref={this.setRef} className={classNames('detailed-status', `detailed-status-${status.get('visibility')}`, { compact })}>
@@ -252,7 +262,7 @@ class DetailedStatus extends ImmutablePureComponent {
           <div className='detailed-status__meta'>
             <a className='detailed-status__datetime' href={status.get('url')} target='_blank' rel='noopener noreferrer'>
               <FormattedDate value={new Date(status.get('created_at'))} hour12={false} year='numeric' month='short' day='2-digit' hour='2-digit' minute='2-digit' />
-            </a>{visibilityLink}{applicationLink}{reblogLink} · {favouriteLink}
+            </a>{edited}{visibilityLink}{applicationLink}{reblogLink} · {favouriteLink}
           </div>
         </div>
       </div>
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 0a62e6b82..02b3473a9 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -967,6 +967,17 @@
   }
 }
 
+.status__content__edited-label {
+  display: block;
+  cursor: default;
+  font-size: 15px;
+  line-height: 20px;
+  padding: 0;
+  padding-top: 8px;
+  color: $dark-text-color;
+  font-weight: 500;
+}
+
 .status__content__spoiler-link {
   display: inline-block;
   border-radius: 2px;
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 3aeecb4ec..706960f92 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -94,49 +94,6 @@ class ActivityPub::Activity
     equals_or_includes_any?(@object['type'], CONVERTED_TYPES)
   end
 
-  def distribute(status)
-    crawl_links(status)
-
-    notify_about_reblog(status) if reblog_of_local_account?(status) && !reblog_by_following_group_account?(status)
-    notify_about_mentions(status)
-
-    # Only continue if the status is supposed to have arrived in real-time.
-    # Note that if @options[:override_timestamps] isn't set, the status
-    # may have a lower snowflake id than other existing statuses, potentially
-    # "hiding" it from paginated API calls
-    return unless @options[:override_timestamps] || status.within_realtime_window?
-
-    distribute_to_followers(status)
-  end
-
-  def reblog_of_local_account?(status)
-    status.reblog? && status.reblog.account.local?
-  end
-
-  def reblog_by_following_group_account?(status)
-    status.reblog? && status.account.group? && status.reblog.account.following?(status.account)
-  end
-
-  def notify_about_reblog(status)
-    NotifyService.new.call(status.reblog.account, :reblog, status)
-  end
-
-  def notify_about_mentions(status)
-    status.active_mentions.includes(:account).each do |mention|
-      next unless mention.account.local? && audience_includes?(mention.account)
-      NotifyService.new.call(mention.account, :mention, mention)
-    end
-  end
-
-  def crawl_links(status)
-    # Spread out crawling randomly to avoid DDoSing the link
-    LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
-  end
-
-  def distribute_to_followers(status)
-    ::DistributionWorker.perform_async(status.id)
-  end
-
   def delete_arrived_first?(uri)
     redis.exists?("delete_upon_arrival:#{@account.id}:#{uri}")
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 6c5d88d18..1f9319290 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -25,7 +25,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       Trends.tags.register(@status)
       Trends.links.register(@status)
 
-      distribute(@status)
+      distribute
     end
 
     @status
@@ -33,6 +33,22 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
 
   private
 
+  def distribute
+    # Notify the author of the original status if that status is local
+    NotifyService.new.call(@status.reblog.account, :reblog, @status) if reblog_of_local_account?(@status) && !reblog_by_following_group_account?(@status)
+
+    # Distribute into home and list feeds
+    ::DistributionWorker.perform_async(@status.id) if @options[:override_timestamps] || @status.within_realtime_window?
+  end
+
+  def reblog_of_local_account?(status)
+    status.reblog? && status.reblog.account.local?
+  end
+
+  def reblog_by_following_group_account?(status)
+    status.reblog? && status.account.group? && status.reblog.account.following?(status.account)
+  end
+
   def audience_to
     as_array(@json['to']).map { |x| value_or_id(x) }
   end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 8a0dc9d33..a861c34bc 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -69,9 +69,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_status
-    @tags     = []
-    @mentions = []
-    @params   = {}
+    @tags                 = []
+    @mentions             = []
+    @silenced_account_ids = []
+    @params               = {}
 
     process_status_params
     process_tags
@@ -84,10 +85,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
-    distribute(@status)
+    distribute
     forward_for_reply
   end
 
+  def distribute
+    # Spread out crawling randomly to avoid DDoSing the link
+    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id)
+
+    # Distribute into home and list feeds and notify mentioned accounts
+    ::DistributionWorker.perform_async(@status.id, silenced_account_ids: @silenced_account_ids) if @options[:override_timestamps] || @status.within_realtime_window?
+  end
+
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
@@ -95,19 +104,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_status_params
+    @status_parser = ActivityPub::Parser::StatusParser.new(@json, followers_collection: @account.followers_url)
+
     @params = begin
       {
-        uri: object_uri,
-        url: object_url || object_uri,
+        uri: @status_parser.uri,
+        url: @status_parser.url || @status_parser.uri,
         account: @account,
-        text: text_from_content || '',
-        language: detected_language,
-        spoiler_text: converted_object_type? ? '' : (text_from_summary || ''),
-        created_at: @object['published'],
+        text: converted_object_type? ? converted_text : (@status_parser.text || ''),
+        language: @status_parser.language || detected_language,
+        spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''),
+        created_at: @status_parser.created_at,
+        edited_at: @status_parser.edited_at,
         override_timestamps: @options[:override_timestamps],
-        reply: @object['inReplyTo'].present?,
-        sensitive: @account.sensitized? || @object['sensitive'] || false,
-        visibility: visibility_from_audience,
+        reply: @status_parser.reply,
+        sensitive: @account.sensitized? || @status_parser.sensitive || false,
+        visibility: @status_parser.visibility,
         thread: replied_to_status,
         conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
@@ -117,42 +129,40 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_audience
-    (audience_to + audience_cc).uniq.each do |audience|
-      next if ActivityPub::TagManager.instance.public_collection?(audience)
+    # Unlike with tags, there is no point in resolving accounts we don't already
+    # know here, because silent mentions would only be used for local access control anyway
+    accounts_in_audience = (audience_to + audience_cc).uniq.filter_map do |audience|
+      account_from_uri(audience) unless ActivityPub::TagManager.instance.public_collection?(audience)
+    end
 
-      # Unlike with tags, there is no point in resolving accounts we don't already
-      # know here, because silent mentions would only be used for local access
-      # control anyway
-      account = account_from_uri(audience)
+    # If the payload was delivered to a specific inbox, the inbox owner must have
+    # access to it, unless they already have access to it anyway
+    if @options[:delivered_to_account_id]
+      accounts_in_audience << delivered_to_account
+      accounts_in_audience.uniq!
+    end
 
-      next if account.nil? || @mentions.any? { |mention| mention.account_id == account.id }
+    accounts_in_audience.each do |account|
+      # This runs after tags are processed, and those translate into non-silent
+      # mentions, which take precedence
+      next if @mentions.any? { |mention| mention.account_id == account.id }
 
       @mentions << Mention.new(account: account, silent: true)
 
       # If there is at least one silent mention, then the status can be considered
       # as a limited-audience status, and not strictly a direct message, but only
       # if we considered a direct message in the first place
-      next unless @params[:visibility] == :direct
-
-      @params[:visibility] = :limited
+      @params[:visibility] = :limited if @params[:visibility] == :direct
     end
 
-    # If the payload was delivered to a specific inbox, the inbox owner must have
-    # access to it, unless they already have access to it anyway
-    return if @options[:delivered_to_account_id].nil? || @mentions.any? { |mention| mention.account_id == @options[:delivered_to_account_id] }
-
-    @mentions << Mention.new(account_id: @options[:delivered_to_account_id], silent: true)
-
-    return unless @params[:visibility] == :direct
-
-    @params[:visibility] = :limited
+    # Accounts that are tagged but are not in the audience are not
+    # supposed to be notified explicitly
+    @silenced_account_ids = @mentions.map(&:account_id) - accounts_in_audience.map(&:id)
   end
 
   def postprocess_audience_and_deliver
     return if @status.mentions.find_by(account_id: @options[:delivered_to_account_id])
 
-    delivered_to_account = Account.find(@options[:delivered_to_account_id])
-
     @status.mentions.create(account: delivered_to_account, silent: true)
     @status.update(visibility: :limited) if @status.direct_visibility?
 
@@ -161,6 +171,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     FeedInsertWorker.perform_async(@status.id, delivered_to_account.id, :home)
   end
 
+  def delivered_to_account
+    @delivered_to_account ||= Account.find(@options[:delivered_to_account_id])
+  end
+
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
@@ -215,21 +229,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def process_emoji(tag)
     return if skip_download?
-    return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
 
-    shortcode = tag['name'].delete(':')
-    image_url = tag['icon']['url']
-    uri       = tag['id']
-    updated   = tag['updated']
-    emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
+    custom_emoji_parser = ActivityPub::Parser::CustomEmojiParser.new(tag)
+
+    return if custom_emoji_parser.shortcode.blank? || custom_emoji_parser.image_remote_url.blank?
 
-    return unless emoji.nil? || image_url != emoji.image_remote_url || (updated && updated >= emoji.updated_at)
+    emoji = CustomEmoji.find_by(shortcode: custom_emoji_parser.shortcode, domain: @account.domain)
 
-    emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
-    emoji.image_remote_url = image_url
-    emoji.save
-  rescue Seahorse::Client::NetworkingError => e
-    Rails.logger.warn "Error storing emoji: #{e}"
+    return unless emoji.nil? || custom_emoji_parser.image_remote_url != emoji.image_remote_url || (custom_emoji_parser.updated_at && custom_emoji_parser.updated_at >= emoji.updated_at)
+
+    begin
+      emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: custom_emoji_parser.shortcode, uri: custom_emoji_parser.uri)
+      emoji.image_remote_url = custom_emoji_parser.image_remote_url
+      emoji.save
+    rescue Seahorse::Client::NetworkingError => e
+      Rails.logger.warn "Error storing emoji: #{e}"
+    end
   end
 
   def process_attachments
@@ -238,14 +253,23 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     media_attachments = []
 
     as_array(@object['attachment']).each do |attachment|
-      next if attachment['url'].blank? || media_attachments.size >= 4
+      media_attachment_parser = ActivityPub::Parser::MediaAttachmentParser.new(attachment)
+
+      next if media_attachment_parser.remote_url.blank? || media_attachments.size >= 4
 
       begin
-        href             = Addressable::URI.parse(attachment['url']).normalize.to_s
-        media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
+        media_attachment = MediaAttachment.create(
+          account: @account,
+          remote_url: media_attachment_parser.remote_url,
+          thumbnail_remote_url: media_attachment_parser.thumbnail_remote_url,
+          description: media_attachment_parser.description,
+          focus: media_attachment_parser.focus,
+          blurhash: media_attachment_parser.blurhash
+        )
+
         media_attachments << media_attachment
 
-        next if unsupported_media_type?(attachment['mediaType']) || skip_download?
+        next if unsupported_media_type?(media_attachment_parser.file_content_type) || skip_download?
 
         media_attachment.download_file!
         media_attachment.download_thumbnail!
@@ -263,42 +287,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     media_attachments
   end
 
-  def icon_url_from_attachment(attachment)
-    url = attachment['icon'].is_a?(Hash) ? attachment['icon']['url'] : attachment['icon']
-    Addressable::URI.parse(url).normalize.to_s if url.present?
-  rescue Addressable::URI::InvalidURIError
-    nil
-  end
-
   def process_poll
-    return unless @object['type'] == 'Question' && (@object['anyOf'].is_a?(Array) || @object['oneOf'].is_a?(Array))
-
-    expires_at = begin
-      if @object['closed'].is_a?(String)
-        @object['closed']
-      elsif !@object['closed'].nil? && !@object['closed'].is_a?(FalseClass)
-        Time.now.utc
-      else
-        @object['endTime']
-      end
-    end
-
-    if @object['anyOf'].is_a?(Array)
-      multiple = true
-      items    = @object['anyOf']
-    else
-      multiple = false
-      items    = @object['oneOf']
-    end
+    poll_parser = ActivityPub::Parser::PollParser.new(@object)
 
-    voters_count = @object['votersCount']
+    return unless poll_parser.valid?
 
     @account.polls.new(
-      multiple: multiple,
-      expires_at: expires_at,
-      options: items.map { |item| item['name'].presence || item['content'] }.compact,
-      cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
-      voters_count: voters_count
+      multiple: poll_parser.multiple,
+      expires_at: poll_parser.expires_at,
+      options: poll_parser.options,
+      cached_tallies: poll_parser.cached_tallies,
+      voters_count: poll_parser.voters_count
     )
   end
 
@@ -351,23 +350,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     end
   end
 
-  def visibility_from_audience
-    if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
-      :public
-    elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
-      :unlisted
-    elsif audience_to.include?(@account.followers_url)
-      :private
-    else
-      :direct
-    end
-  end
-
-  def audience_includes?(account)
-    uri = ActivityPub::TagManager.instance.uri_for(account)
-    audience_to.include?(uri) || audience_cc.include?(uri)
-  end
-
   def replied_to_status
     return @replied_to_status if defined?(@replied_to_status)
 
@@ -384,81 +366,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     value_or_id(@object['inReplyTo'])
   end
 
-  def text_from_content
-    return Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type?
-
-    if @object['content'].present?
-      @object['content']
-    elsif content_language_map?
-      @object['contentMap'].values.first
-    end
-  end
-
-  def text_from_summary
-    if @object['summary'].present?
-      @object['summary']
-    elsif summary_language_map?
-      @object['summaryMap'].values.first
-    end
-  end
-
-  def text_from_name
-    if @object['name'].present?
-      @object['name']
-    elsif name_language_map?
-      @object['nameMap'].values.first
-    end
+  def converted_text
+    Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n"))
   end
 
   def detected_language
-    if content_language_map?
-      @object['contentMap'].keys.first
-    elsif name_language_map?
-      @object['nameMap'].keys.first
-    elsif summary_language_map?
-      @object['summaryMap'].keys.first
-    elsif supported_object_type?
-      LanguageDetector.instance.detect(text_from_content, @account)
-    end
-  end
-
-  def object_url
-    return if @object['url'].blank?
-
-    url_candidate = url_to_href(@object['url'], 'text/html')
-
-    if invalid_origin?(url_candidate)
-      nil
-    else
-      url_candidate
-    end
-  end
-
-  def summary_language_map?
-    @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty?
-  end
-
-  def content_language_map?
-    @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
-  end
-
-  def name_language_map?
-    @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty?
+    LanguageDetector.instance.detect(@status_parser.text, @account) if supported_object_type?
   end
 
   def unsupported_media_type?(mime_type)
     mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type)
   end
 
-  def supported_blurhash?(blurhash)
-    components = blurhash.blank? || !blurhash_valid_chars?(blurhash) ? nil : Blurhash.components(blurhash)
-    components.present? && components.none? { |comp| comp > 5 }
-  end
-
-  def blurhash_valid_chars?(blurhash)
-    /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
-  end
-
   def skip_download?
     return @skip_download if defined?(@skip_download)
 
diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb
index 018e2df54..f04ad321b 100644
--- a/app/lib/activitypub/activity/update.rb
+++ b/app/lib/activitypub/activity/update.rb
@@ -1,32 +1,31 @@
 # frozen_string_literal: true
 
 class ActivityPub::Activity::Update < ActivityPub::Activity
-  SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
-
   def perform
     dereference_object!
 
-    if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES)
+    if equals_or_includes_any?(@object['type'], %w(Application Group Organization Person Service))
       update_account
-    elsif equals_or_includes_any?(@object['type'], %w(Question))
-      update_poll
+    elsif equals_or_includes_any?(@object['type'], %w(Note Question))
+      update_status
     end
   end
 
   private
 
   def update_account
-    return if @account.uri != object_uri
+    return reject_payload! if @account.uri != object_uri
 
     ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true)
   end
 
-  def update_poll
+  def update_status
     return reject_payload! if invalid_origin?(@object['id'])
 
     status = Status.find_by(uri: object_uri, account_id: @account.id)
-    return if status.nil? || status.preloadable_poll.nil?
 
-    ActivityPub::ProcessPollService.new.call(status.preloadable_poll, @object)
+    return if status.nil?
+
+    ActivityPub::ProcessStatusUpdateService.new.call(status, @object)
   end
 end
diff --git a/app/lib/activitypub/parser/custom_emoji_parser.rb b/app/lib/activitypub/parser/custom_emoji_parser.rb
new file mode 100644
index 000000000..724c60215
--- /dev/null
+++ b/app/lib/activitypub/parser/custom_emoji_parser.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ActivityPub::Parser::CustomEmojiParser
+  include JsonLdHelper
+
+  def initialize(json)
+    @json = json
+  end
+
+  def uri
+    @json['id']
+  end
+
+  def shortcode
+    @json['name']&.delete(':')
+  end
+
+  def image_remote_url
+    @json.dig('icon', 'url')
+  end
+
+  def updated_at
+    @json['updated']&.to_datetime
+  rescue ArgumentError
+    nil
+  end
+end
diff --git a/app/lib/activitypub/parser/media_attachment_parser.rb b/app/lib/activitypub/parser/media_attachment_parser.rb
new file mode 100644
index 000000000..1798e58a4
--- /dev/null
+++ b/app/lib/activitypub/parser/media_attachment_parser.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class ActivityPub::Parser::MediaAttachmentParser
+  include JsonLdHelper
+
+  def initialize(json)
+    @json = json
+  end
+
+  # @param [MediaAttachment] previous_record
+  def significantly_changes?(previous_record)
+    remote_url != previous_record.remote_url ||
+      thumbnail_remote_url != previous_record.thumbnail_remote_url ||
+      description != previous_record.description
+  end
+
+  def remote_url
+    Addressable::URI.parse(@json['url'])&.normalize&.to_s
+  rescue Addressable::URI::InvalidURIError
+    nil
+  end
+
+  def thumbnail_remote_url
+    Addressable::URI.parse(@json['icon'].is_a?(Hash) ? @json['icon']['url'] : @json['icon'])&.normalize&.to_s
+  rescue Addressable::URI::InvalidURIError
+    nil
+  end
+
+  def description
+    @json['summary'].presence || @json['name'].presence
+  end
+
+  def focus
+    @json['focalPoint']
+  end
+
+  def blurhash
+    supported_blurhash? ? @json['blurhash'] : nil
+  end
+
+  def file_content_type
+    @json['mediaType']
+  end
+
+  private
+
+  def supported_blurhash?
+    components = begin
+      blurhash = @json['blurhash']
+
+      if blurhash.present? && /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
+        Blurhash.components(blurhash)
+      end
+    end
+
+    components.present? && components.none? { |comp| comp > 5 }
+  end
+end
diff --git a/app/lib/activitypub/parser/poll_parser.rb b/app/lib/activitypub/parser/poll_parser.rb
new file mode 100644
index 000000000..758c03f07
--- /dev/null
+++ b/app/lib/activitypub/parser/poll_parser.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class ActivityPub::Parser::PollParser
+  include JsonLdHelper
+
+  def initialize(json)
+    @json = json
+  end
+
+  def valid?
+    equals_or_includes?(@json['type'], 'Question') && items.is_a?(Array)
+  end
+
+  # @param [Poll] previous_record
+  def significantly_changes?(previous_record)
+    options != previous_record.options ||
+      multiple != previous_record.multiple
+  end
+
+  def options
+    items.filter_map { |item| item['name'].presence || item['content'] }
+  end
+
+  def multiple
+    @json['anyOf'].is_a?(Array)
+  end
+
+  def expires_at
+    if @json['closed'].is_a?(String)
+      @json['closed'].to_datetime
+    elsif !@json['closed'].nil? && !@json['closed'].is_a?(FalseClass)
+      Time.now.utc
+    else
+      @json['endTime']&.to_datetime
+    end
+  rescue ArgumentError
+    nil
+  end
+
+  def voters_count
+    @json['votersCount']
+  end
+
+  def cached_tallies
+    items.map { |item| item.dig('replies', 'totalItems') || 0 }
+  end
+
+  private
+
+  def items
+    @json['anyOf'] || @json['oneOf']
+  end
+end
diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb
new file mode 100644
index 000000000..3ba154d01
--- /dev/null
+++ b/app/lib/activitypub/parser/status_parser.rb
@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+class ActivityPub::Parser::StatusParser
+  include JsonLdHelper
+
+  # @param [Hash] json
+  # @param [Hash] magic_values
+  # @option magic_values [String] :followers_collection
+  def initialize(json, magic_values = {})
+    @json         = json
+    @object       = json['object'] || json
+    @magic_values = magic_values
+  end
+
+  def uri
+    id = @object['id']
+
+    if id&.start_with?('bear:')
+      Addressable::URI.parse(id).query_values['u']
+    else
+      id
+    end
+  rescue Addressable::URI::InvalidURIError
+    id
+  end
+
+  def url
+    url_to_href(@object['url'], 'text/html') if @object['url'].present?
+  end
+
+  def text
+    if @object['content'].present?
+      @object['content']
+    elsif content_language_map?
+      @object['contentMap'].values.first
+    end
+  end
+
+  def spoiler_text
+    if @object['summary'].present?
+      @object['summary']
+    elsif summary_language_map?
+      @object['summaryMap'].values.first
+    end
+  end
+
+  def title
+    if @object['name'].present?
+      @object['name']
+    elsif name_language_map?
+      @object['nameMap'].values.first
+    end
+  end
+
+  def created_at
+    @object['published']&.to_datetime
+  rescue ArgumentError
+    nil
+  end
+
+  def edited_at
+    @object['updated']&.to_datetime
+  rescue ArgumentError
+    nil
+  end
+
+  def reply
+    @object['inReplyTo'].present?
+  end
+
+  def sensitive
+    @object['sensitive']
+  end
+
+  def visibility
+    if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
+      :public
+    elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
+      :unlisted
+    elsif audience_to.include?(@magic_values[:followers_collection])
+      :private
+    else
+      :direct
+    end
+  end
+
+  def language
+    if content_language_map?
+      @object['contentMap'].keys.first
+    elsif name_language_map?
+      @object['nameMap'].keys.first
+    elsif summary_language_map?
+      @object['summaryMap'].keys.first
+    end
+  end
+
+  private
+
+  def audience_to
+    as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) }
+  end
+
+  def audience_cc
+    as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) }
+  end
+
+  def summary_language_map?
+    @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty?
+  end
+
+  def content_language_map?
+    @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
+  end
+
+  def name_language_map?
+    @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty?
+  end
+end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index d5e435216..c4dd9d00f 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -53,46 +53,50 @@ class FeedManager
   # Add a status to a home feed and send a streaming API update
   # @param [Account] account
   # @param [Status] status
+  # @param [Boolean] update
   # @return [Boolean]
-  def push_to_home(account, status)
+  def push_to_home(account, status, update: false)
     return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
 
     trim(:home, account.id)
-    PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
+    PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}", update: update) if push_update_required?("timeline:#{account.id}")
     true
   end
 
   # Remove a status from a home feed and send a streaming API update
   # @param [Account] account
   # @param [Status] status
+  # @param [Boolean] update
   # @return [Boolean]
-  def unpush_from_home(account, status)
+  def unpush_from_home(account, status, update: false)
     return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
 
-    redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
+    redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update
     true
   end
 
   # Add a status to a list feed and send a streaming API update
   # @param [List] list
   # @param [Status] status
+  # @param [Boolean] update
   # @return [Boolean]
-  def push_to_list(list, status)
+  def push_to_list(list, status, update: false)
     return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
 
     trim(:list, list.id)
-    PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
+    PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", update: update) if push_update_required?("timeline:list:#{list.id}")
     true
   end
 
   # Remove a status from a list feed and send a streaming API update
   # @param [List] list
   # @param [Status] status
+  # @param [Boolean] update
   # @return [Boolean]
-  def unpush_from_list(list, status)
+  def unpush_from_list(list, status, update: false)
     return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
 
-    redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
+    redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update
     true
   end
 
diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb
index 735d66a4f..98e502bb6 100644
--- a/app/lib/status_reach_finder.rb
+++ b/app/lib/status_reach_finder.rb
@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
 class StatusReachFinder
-  def initialize(status)
-    @status = status
+  # @param [Status] status
+  # @param [Hash] options
+  # @option options [Boolean] :unsafe
+  def initialize(status, options = {})
+    @status  = status
+    @options = options
   end
 
   def inboxes
@@ -38,7 +42,7 @@ class StatusReachFinder
   end
 
   def replied_to_account_id
-    @status.in_reply_to_account_id
+    @status.in_reply_to_account_id if distributable?
   end
 
   def reblog_of_account_id
@@ -49,21 +53,26 @@ class StatusReachFinder
     @status.mentions.pluck(:account_id)
   end
 
+  # Beware: Reblogs can be created without the author having had access to the status
   def reblogs_account_ids
-    @status.reblogs.pluck(:account_id)
+    @status.reblogs.pluck(:account_id) if distributable? || unsafe?
   end
 
+  # Beware: Favourites can be created without the author having had access to the status
   def favourites_account_ids
-    @status.favourites.pluck(:account_id)
+    @status.favourites.pluck(:account_id) if distributable? || unsafe?
   end
 
+  # Beware: Replies can be created without the author having had access to the status
   def replies_account_ids
-    @status.replies.pluck(:account_id)
+    @status.replies.pluck(:account_id) if distributable? || unsafe?
   end
 
   def followers_inboxes
-    if @status.in_reply_to_local_account? && @status.distributable?
+    if @status.in_reply_to_local_account? && distributable?
       @status.account.followers.or(@status.thread.account.followers).inboxes
+    elsif @status.direct_visibility? || @status.limited_visibility?
+      []
     else
       @status.account.followers.inboxes
     end
@@ -76,4 +85,12 @@ class StatusReachFinder
       []
     end
   end
+
+  def distributable?
+    @status.public_visibility? || @status.unlisted_visibility?
+  end
+
+  def unsafe?
+    @options[:unsafe]
+  end
 end
diff --git a/app/models/poll.rb b/app/models/poll.rb
index d2a17277b..71b5e191f 100644
--- a/app/models/poll.rb
+++ b/app/models/poll.rb
@@ -26,6 +26,7 @@ class Poll < ApplicationRecord
   belongs_to :status
 
   has_many :votes, class_name: 'PollVote', inverse_of: :poll, dependent: :delete_all
+  has_many :voters, -> { group('accounts.id') }, through: :votes, class_name: 'Account', source: :account
 
   has_many :notifications, as: :activity, dependent: :destroy
 
diff --git a/app/models/status.rb b/app/models/status.rb
index 749a23718..3358d6891 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -23,6 +23,7 @@
 #  in_reply_to_account_id :bigint(8)
 #  poll_id                :bigint(8)
 #  deleted_at             :datetime
+#  edited_at              :datetime
 #
 
 class Status < ApplicationRecord
@@ -56,6 +57,8 @@ class Status < ApplicationRecord
   belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies, optional: true
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
+  has_many :edits, class_name: 'StatusEdit', inverse_of: :status, dependent: :destroy
+
   has_many :favourites, inverse_of: :status, dependent: :destroy
   has_many :bookmarks, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
@@ -209,6 +212,10 @@ class Status < ApplicationRecord
     public_visibility? || unlisted_visibility?
   end
 
+  def edited?
+    edited_at.present?
+  end
+
   alias sign? distributable?
 
   def with_media?
diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb
new file mode 100644
index 000000000..a89df86c5
--- /dev/null
+++ b/app/models/status_edit.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: status_edits
+#
+#  id                        :bigint(8)        not null, primary key
+#  status_id                 :bigint(8)        not null
+#  account_id                :bigint(8)
+#  text                      :text             default(""), not null
+#  spoiler_text              :text             default(""), not null
+#  media_attachments_changed :boolean          default(FALSE), not null
+#  created_at                :datetime         not null
+#  updated_at                :datetime         not null
+#
+
+class StatusEdit < ApplicationRecord
+  belongs_to :status
+  belongs_to :account, optional: true
+
+  default_scope { order(id: :asc) }
+
+  delegate :local?, to: :status
+end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 7c52b634d..12dabc65a 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -11,6 +11,7 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
 
   attribute :content
   attribute :content_map, if: :language?
+  attribute :updated, if: :edited?
 
   has_many :media_attachments, key: :attachment
   has_many :virtual_tags, key: :tag
@@ -65,6 +66,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     object.language.present?
   end
 
+  delegate :edited?, to: :object
+
   def in_reply_to
     return unless object.reply? && !object.thread.nil?
 
@@ -79,6 +82,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     object.created_at.iso8601
   end
 
+  def updated
+    object.edited_at.iso8601
+  end
+
   def url
     ActivityPub::TagManager.instance.url_for(object)
   end
diff --git a/app/serializers/rest/status_edit_serializer.rb b/app/serializers/rest/status_edit_serializer.rb
new file mode 100644
index 000000000..b123b4e09
--- /dev/null
+++ b/app/serializers/rest/status_edit_serializer.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class REST::StatusEditSerializer < ActiveModel::Serializer
+  attributes :text, :spoiler_text, :media_attachments_changed,
+             :created_at
+end
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index e84f3bd61..aef51e0f7 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
              :sensitive, :spoiler_text, :visibility, :language,
              :uri, :url, :replies_count, :reblogs_count,
-             :favourites_count
+             :favourites_count, :edited_at
 
   attribute :favourited, if: :current_user?
   attribute :reblogged, if: :current_user?
diff --git a/app/serializers/rest/status_source_serializer.rb b/app/serializers/rest/status_source_serializer.rb
new file mode 100644
index 000000000..cd3c74084
--- /dev/null
+++ b/app/serializers/rest/status_source_serializer.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class REST::StatusSourceSerializer < ActiveModel::Serializer
+  attributes :id, :text, :spoiler_text
+
+  def id
+    object.id.to_s
+  end
+end
diff --git a/app/services/activitypub/fetch_remote_poll_service.rb b/app/services/activitypub/fetch_remote_poll_service.rb
index 1c79ecf11..1829e791c 100644
--- a/app/services/activitypub/fetch_remote_poll_service.rb
+++ b/app/services/activitypub/fetch_remote_poll_service.rb
@@ -8,6 +8,6 @@ class ActivityPub::FetchRemotePollService < BaseService
 
     return unless supported_context?(json)
 
-    ActivityPub::ProcessPollService.new.call(poll, json)
+    ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json)
   end
 end
diff --git a/app/services/activitypub/process_poll_service.rb b/app/services/activitypub/process_poll_service.rb
deleted file mode 100644
index d83e614d8..000000000
--- a/app/services/activitypub/process_poll_service.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-class ActivityPub::ProcessPollService < BaseService
-  include JsonLdHelper
-
-  def call(poll, json)
-    @json = json
-
-    return unless expected_type?
-
-    previous_expires_at = poll.expires_at
-
-    expires_at = begin
-      if @json['closed'].is_a?(String)
-        @json['closed']
-      elsif !@json['closed'].nil? && !@json['closed'].is_a?(FalseClass)
-        Time.now.utc
-      else
-        @json['endTime']
-      end
-    end
-
-    items = begin
-      if @json['anyOf'].is_a?(Array)
-        @json['anyOf']
-      else
-        @json['oneOf']
-      end
-    end
-
-    voters_count = @json['votersCount']
-
-    latest_options = items.filter_map { |item| item['name'].presence || item['content'] }
-
-    # If for some reasons the options were changed, it invalidates all previous
-    # votes, so we need to remove them
-    poll.votes.delete_all if latest_options != poll.options
-
-    begin
-      poll.update!(
-        last_fetched_at: Time.now.utc,
-        expires_at: expires_at,
-        options: latest_options,
-        cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
-        voters_count: voters_count
-      )
-    rescue ActiveRecord::StaleObjectError
-      poll.reload
-      retry
-    end
-
-    # If the poll had no expiration date set but now has, and people have voted,
-    # schedule a notification.
-    if previous_expires_at.nil? && poll.expires_at.present? && poll.votes.exists?
-      PollExpirationNotifyWorker.perform_at(poll.expires_at + 5.minutes, poll.id)
-    end
-  end
-
-  private
-
-  def expected_type?
-    equals_or_includes_any?(@json['type'], %w(Question))
-  end
-end
diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb
new file mode 100644
index 000000000..e3e9b9d6a
--- /dev/null
+++ b/app/services/activitypub/process_status_update_service.rb
@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+class ActivityPub::ProcessStatusUpdateService < BaseService
+  include JsonLdHelper
+
+  def call(status, json)
+    @json                      = json
+    @status_parser             = ActivityPub::Parser::StatusParser.new(@json)
+    @uri                       = @status_parser.uri
+    @status                    = status
+    @account                   = status.account
+    @media_attachments_changed = false
+
+    # Only native types can be updated at the moment
+    return if !expected_type? || already_updated_more_recently?
+
+    # Only allow processing one create/update per status at a time
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        Status.transaction do
+          create_previous_edit!
+          update_media_attachments!
+          update_poll!
+          update_immediate_attributes!
+          update_metadata!
+          create_edit!
+        end
+
+        queue_poll_notifications!
+        reset_preview_card!
+        broadcast_updates!
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
+  end
+
+  private
+
+  def update_media_attachments!
+    previous_media_attachments = @status.media_attachments.to_a
+    next_media_attachments     = []
+
+    as_array(@json['attachment']).each do |attachment|
+      media_attachment_parser = ActivityPub::Parser::MediaAttachmentParser.new(attachment)
+
+      next if media_attachment_parser.remote_url.blank? || next_media_attachments.size > 4
+
+      begin
+        media_attachment   = previous_media_attachments.find { |previous_media_attachment| previous_media_attachment.remote_url == media_attachment_parser.remote_url }
+        media_attachment ||= MediaAttachment.new(account: @account, remote_url: media_attachment_parser.remote_url)
+
+        # If a previously existing media attachment was significantly updated, mark
+        # media attachments as changed even if none were added or removed
+        if media_attachment_parser.significantly_changes?(media_attachment)
+          @media_attachments_changed = true
+        end
+
+        media_attachment.description          = media_attachment_parser.description
+        media_attachment.focus                = media_attachment_parser.focus
+        media_attachment.thumbnail_remote_url = media_attachment_parser.thumbnail_remote_url
+        media_attachment.blurhash             = media_attachment_parser.blurhash
+        media_attachment.save!
+
+        next_media_attachments << media_attachment
+
+        next if unsupported_media_type?(media_attachment_parser.file_content_type) || skip_download?
+
+        RedownloadMediaWorker.perform_async(media_attachment.id) if media_attachment.remote_url_previously_changed? || media_attachment.thumbnail_remote_url_previously_changed?
+      rescue Addressable::URI::InvalidURIError => e
+        Rails.logger.debug "Invalid URL in attachment: #{e}"
+      end
+    end
+
+    removed_media_attachments = previous_media_attachments - next_media_attachments
+    added_media_attachments   = next_media_attachments - previous_media_attachments
+
+    MediaAttachment.where(id: removed_media_attachments.map(&:id)).update_all(status_id: nil)
+    MediaAttachment.where(id: added_media_attachments.map(&:id)).update_all(status_id: @status.id)
+
+    @media_attachments_changed = true if removed_media_attachments.positive? || added_media_attachments.positive?
+  end
+
+  def update_poll!
+    previous_poll        = @status.preloadable_poll
+    @previous_expires_at = previous_poll&.expires_at
+    poll_parser          = ActivityPub::Parser::PollParser.new(@json)
+
+    if poll_parser.valid?
+      poll = previous_poll || @account.polls.new(status: @status)
+
+      # If for some reasons the options were changed, it invalidates all previous
+      # votes, so we need to remove them
+      if poll_parser.significantly_changes?(poll)
+        @media_attachments_changed = true
+        poll.votes.delete_all unless poll.new_record?
+      end
+
+      poll.last_fetched_at = Time.now.utc
+      poll.options         = poll_parser.options
+      poll.multiple        = poll_parser.multiple
+      poll.expires_at      = poll_parser.expires_at
+      poll.voters_count    = poll_parser.voters_count
+      poll.cached_tallies  = poll_parser.cached_tallies
+      poll.save!
+
+      @status.poll_id = poll.id
+    elsif previous_poll.present?
+      previous_poll.destroy!
+      @media_attachments_changed = true
+      @status.poll_id = nil
+    end
+  end
+
+  def update_immediate_attributes!
+    @status.text         = @status_parser.text || ''
+    @status.spoiler_text = @status_parser.spoiler_text || ''
+    @status.sensitive    = @account.sensitized? || @status_parser.sensitive || false
+    @status.language     = @status_parser.language || detected_language
+    @status.edited_at    = @status_parser.edited_at || Time.now.utc
+
+    @status.save!
+  end
+
+  def update_metadata!
+    @raw_tags     = []
+    @raw_mentions = []
+    @raw_emojis   = []
+
+    as_array(@json['tag']).each do |tag|
+      if equals_or_includes?(tag['type'], 'Hashtag')
+        @raw_tags << tag['name']
+      elsif equals_or_includes?(tag['type'], 'Mention')
+        @raw_mentions << tag['href']
+      elsif equals_or_includes?(tag['type'], 'Emoji')
+        @raw_emojis << tag
+      end
+    end
+
+    update_tags!
+    update_mentions!
+    update_emojis!
+  end
+
+  def update_tags!
+    @status.tags = Tag.find_or_create_by_names(@raw_tags)
+  end
+
+  def update_mentions!
+    previous_mentions = @status.active_mentions.includes(:account).to_a
+    current_mentions  = []
+
+    @raw_mentions.each do |href|
+      next if href.blank?
+
+      account   = ActivityPub::TagManager.instance.uri_to_resource(href, Account)
+      account ||= ActivityPub::FetchRemoteAccountService.new.call(href)
+
+      next if account.nil?
+
+      mention   = previous_mentions.find { |x| x.account_id == account.id }
+      mention ||= account.mentions.new(status: @status)
+
+      current_mentions << mention
+    end
+
+    current_mentions.each do |mention|
+      mention.save if mention.new_record?
+    end
+
+    # If previous mentions are no longer contained in the text, convert them
+    # to silent mentions, since withdrawing access from someone who already
+    # received a notification might be more confusing
+    removed_mentions = previous_mentions - current_mentions
+
+    Mention.where(id: removed_mentions.map(&:id)).update_all(silent: true) unless removed_mentions.empty?
+  end
+
+  def update_emojis!
+    return if skip_download?
+
+    @raw_emojis.each do |raw_emoji|
+      custom_emoji_parser = ActivityPub::Parser::CustomEmojiParser.new(raw_emoji)
+
+      next if custom_emoji_parser.shortcode.blank? || custom_emoji_parser.image_remote_url.blank?
+
+      emoji = CustomEmoji.find_by(shortcode: custom_emoji_parser.shortcode, domain: @account.domain)
+
+      next unless emoji.nil? || custom_emoji_parser.image_remote_url != emoji.image_remote_url || (custom_emoji_parser.updated_at && custom_emoji_parser.updated_at >= emoji.updated_at)
+
+      begin
+        emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: custom_emoji_parser.shortcode, uri: custom_emoji_parser.uri)
+        emoji.image_remote_url = custom_emoji_parser.image_remote_url
+        emoji.save
+      rescue Seahorse::Client::NetworkingError => e
+        Rails.logger.warn "Error storing emoji: #{e}"
+      end
+    end
+  end
+
+  def expected_type?
+    equals_or_includes_any?(@json['type'], %w(Note Question))
+  end
+
+  def lock_options
+    { redis: Redis.current, key: "create:#{@uri}", autorelease: 15.minutes.seconds }
+  end
+
+  def detected_language
+    LanguageDetector.instance.detect(@status_parser.text, @account)
+  end
+
+  def create_previous_edit!
+    # We only need to create a previous edit when no previous edits exist, e.g.
+    # when the status has never been edited. For other cases, we always create
+    # an edit, so the step can be skipped
+
+    return if @status.edits.any?
+
+    @status.edits.create(
+      text: @status.text,
+      spoiler_text: @status.spoiler_text,
+      media_attachments_changed: false,
+      account_id: @account.id,
+      created_at: @status.created_at
+    )
+  end
+
+  def create_edit!
+    return unless @status.text_previously_changed? || @status.spoiler_text_previously_changed? || @media_attachments_changed
+
+    @status_edit = @status.edits.create(
+      text: @status.text,
+      spoiler_text: @status.spoiler_text,
+      media_attachments_changed: @media_attachments_changed,
+      account_id: @account.id,
+      created_at: @status.edited_at
+    )
+  end
+
+  def skip_download?
+    return @skip_download if defined?(@skip_download)
+
+    @skip_download ||= DomainBlock.reject_media?(@account.domain)
+  end
+
+  def unsupported_media_type?(mime_type)
+    mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type)
+  end
+
+  def already_updated_more_recently?
+    @status.edited_at.present? && @status_parser.edited_at.present? && @status.edited_at > @status_parser.edited_at
+  end
+
+  def reset_preview_card!
+    @status.preview_cards.clear if @status.text_previously_changed? || @status.spoiler_text.present?
+    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id) if @status.spoiler_text.blank?
+  end
+
+  def broadcast_updates!
+    ::DistributionWorker.perform_async(@status.id, update: true)
+  end
+
+  def queue_poll_notifications!
+    poll = @status.preloadable_poll
+
+    # If the poll had no expiration date set but now has, or now has a sooner
+    # expiration date, and people have voted, schedule a notification
+
+    return unless poll.present? && poll.expires_at.present? && poll.votes.exists?
+
+    PollExpirationNotifyWorker.remove_from_scheduled(poll.id) if @previous_expires_at.present? && @previous_expires_at > poll.expires_at
+    PollExpirationNotifyWorker.perform_at(poll.expires_at + 5.minutes, poll.id)
+  end
+end
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index b72bb82d3..f62f78a79 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -3,107 +3,126 @@
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status
-  def call(status)
-    raise Mastodon::RaceConditionError if status.visibility.nil?
-
-    deliver_to_self(status) if status.account.local?
-
-    if status.direct_visibility?
-      deliver_to_mentioned_followers(status)
-      deliver_to_own_conversation(status)
-    elsif status.limited_visibility?
-      deliver_to_mentioned_followers(status)
-    else
-      deliver_to_followers(status)
-      deliver_to_lists(status)
-    end
+  # @param [Hash] options
+  # @option options [Boolean] update
+  # @option options [Array<Integer>] silenced_account_ids
+  def call(status, options = {})
+    @status    = status
+    @account   = status.account
+    @options   = options
+
+    check_race_condition!
+
+    fan_out_to_local_recipients!
+    fan_out_to_public_streams! if broadcastable?
+  end
 
-    return if status.account.silenced? || !status.public_visibility? || status.reblog?
+  private
 
-    render_anonymous_payload(status)
+  def check_race_condition!
+    # I don't know why but at some point we had an issue where
+    # this service was being executed with status objects
+    # that had a null visibility - which should not be possible
+    # since the column in the database is not nullable.
+    #
+    # This check re-queues the service to be run at a later time
+    # with the full object, if something like it occurs
 
-    deliver_to_hashtags(status)
+    raise Mastodon::RaceConditionError if @status.visibility.nil?
+  end
 
-    return if status.reply? && status.in_reply_to_account_id != status.account_id
+  def fan_out_to_local_recipients!
+    deliver_to_self!
+    notify_mentioned_accounts!
 
-    deliver_to_public(status)
-    deliver_to_media(status) if status.media_attachments.any?
+    case @status.visibility.to_sym
+    when :public, :unlisted, :private
+      deliver_to_all_followers!
+      deliver_to_lists!
+    when :limited
+      deliver_to_mentioned_followers!
+    else
+      deliver_to_mentioned_followers!
+      deliver_to_conversation!
+    end
   end
 
-  private
+  def fan_out_to_public_streams!
+    broadcast_to_hashtag_streams!
+    broadcast_to_public_streams!
+  end
 
-  def deliver_to_self(status)
-    Rails.logger.debug "Delivering status #{status.id} to author"
-    FeedManager.instance.push_to_home(status.account, status)
+  def deliver_to_self!
+    FeedManager.instance.push_to_home(@account, @status, update: update?) if @account.local?
   end
 
-  def deliver_to_followers(status)
-    Rails.logger.debug "Delivering status #{status.id} to followers"
+  def notify_mentioned_accounts!
+    @status.active_mentions.where.not(id: @options[:silenced_account_ids] || []).joins(:account).merge(Account.local).select(:id, :account_id).reorder(nil).find_in_batches do |mentions|
+      LocalNotificationWorker.push_bulk(mentions) do |mention|
+        [mention.account_id, mention.id, 'Mention', :mention]
+      end
+    end
+  end
 
-    status.account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers|
+  def deliver_to_all_followers!
+    @account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers|
       FeedInsertWorker.push_bulk(followers) do |follower|
-        [status.id, follower.id, :home]
+        [@status.id, follower.id, :home, update: update?]
       end
     end
   end
 
-  def deliver_to_lists(status)
-    Rails.logger.debug "Delivering status #{status.id} to lists"
-
-    status.account.lists_for_local_distribution.select(:id).reorder(nil).find_in_batches do |lists|
+  def deliver_to_lists!
+    @account.lists_for_local_distribution.select(:id).reorder(nil).find_in_batches do |lists|
       FeedInsertWorker.push_bulk(lists) do |list|
-        [status.id, list.id, :list]
+        [@status.id, list.id, :list, update: update?]
       end
     end
   end
 
-  def deliver_to_mentioned_followers(status)
-    Rails.logger.debug "Delivering status #{status.id} to limited followers"
-
-    status.mentions.joins(:account).merge(status.account.followers_for_local_distribution).select(:id, :account_id).reorder(nil).find_in_batches do |mentions|
+  def deliver_to_mentioned_followers!
+    @status.mentions.joins(:account).merge(@account.followers_for_local_distribution).select(:id, :account_id).reorder(nil).find_in_batches do |mentions|
       FeedInsertWorker.push_bulk(mentions) do |mention|
-        [status.id, mention.account_id, :home]
+        [@status.id, mention.account_id, :home, update: update?]
       end
     end
   end
 
-  def render_anonymous_payload(status)
-    @payload = InlineRenderer.render(status, nil, :status)
-    @payload = Oj.dump(event: :update, payload: @payload)
+  def broadcast_to_hashtag_streams!
+    @status.tags.pluck(:name).each do |hashtag|
+      Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", anonymous_payload)
+      Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", anonymous_payload) if @status.local?
+    end
   end
 
-  def deliver_to_hashtags(status)
-    Rails.logger.debug "Delivering status #{status.id} to hashtags"
+  def broadcast_to_public_streams!
+    return if @status.reply? && @status.in_reply_to_account_id != @account.id
+
+    Redis.current.publish('timeline:public', anonymous_payload)
+    Redis.current.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', anonymous_payload)
 
-    status.tags.pluck(:name).each do |hashtag|
-      Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload)
-      Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if status.local?
+    if @status.media_attachments.any?
+      Redis.current.publish('timeline:public:media', anonymous_payload)
+      Redis.current.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', anonymous_payload)
     end
   end
 
-  def deliver_to_public(status)
-    Rails.logger.debug "Delivering status #{status.id} to public timeline"
-
-    Redis.current.publish('timeline:public', @payload)
-    if status.local?
-      Redis.current.publish('timeline:public:local', @payload)
-    else
-      Redis.current.publish('timeline:public:remote', @payload)
-    end
+  def deliver_to_conversation!
+    AccountConversation.add_status(@account, @status) unless update?
   end
 
-  def deliver_to_media(status)
-    Rails.logger.debug "Delivering status #{status.id} to media timeline"
+  def anonymous_payload
+    @anonymous_payload ||= Oj.dump(
+      event: update? ? :'status.update' : :update,
+      payload: InlineRenderer.render(@status, nil, :status)
+    )
+  end
 
-    Redis.current.publish('timeline:public:media', @payload)
-    if status.local?
-      Redis.current.publish('timeline:public:local:media', @payload)
-    else
-      Redis.current.publish('timeline:public:remote:media', @payload)
-    end
+  def update?
+    @is_update
   end
 
-  def deliver_to_own_conversation(status)
-    AccountConversation.add_status(status.account, status)
+  def broadcastable?
+    @status.public_visibility? && !@status.reblog? && !@account.silenced?
   end
 end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 73dbb1834..9d239fc65 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -8,12 +8,23 @@ class ProcessMentionsService < BaseService
   # remote users
   # @param [Status] status
   def call(status)
-    return unless status.local?
+    @status = status
 
-    @status  = status
-    mentions = []
+    return unless @status.local?
 
-    status.text = status.text.gsub(Account::MENTION_RE) do |match|
+    @previous_mentions = @status.active_mentions.includes(:account).to_a
+    @current_mentions  = []
+
+    Status.transaction do
+      scan_text!
+      assign_mentions!
+    end
+  end
+
+  private
+
+  def scan_text!
+    @status.text = @status.text.gsub(Account::MENTION_RE) do |match|
       username, domain = Regexp.last_match(1).split('@')
 
       domain = begin
@@ -26,49 +37,45 @@ class ProcessMentionsService < BaseService
 
       mentioned_account = Account.find_remote(username, domain)
 
+      # If the account cannot be found or isn't the right protocol,
+      # first try to resolve it
       if mention_undeliverable?(mentioned_account)
         begin
-          mentioned_account = resolve_account_service.call(Regexp.last_match(1))
+          mentioned_account = ResolveAccountService.new.call(Regexp.last_match(1))
         rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError
           mentioned_account = nil
         end
       end
 
+      # If after resolving it still isn't found or isn't the right
+      # protocol, then give up
       next match if mention_undeliverable?(mentioned_account) || mentioned_account&.suspended?
 
-      mention = mentioned_account.mentions.new(status: status)
-      mentions << mention if mention.save
+      mention   = @previous_mentions.find { |x| x.account_id == mentioned_account.id }
+      mention ||= mentioned_account.mentions.new(status: @status)
+
+      @current_mentions << mention
 
       "@#{mentioned_account.acct}"
     end
 
-    status.save!
-
-    mentions.each { |mention| create_notification(mention) }
+    @status.save!
   end
 
-  private
-
-  def mention_undeliverable?(mentioned_account)
-    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus?)
-  end
-
-  def create_notification(mention)
-    mentioned_account = mention.account
-
-    if mentioned_account.local?
-      LocalNotificationWorker.perform_async(mentioned_account.id, mention.id, mention.class.name, :mention)
-    elsif mentioned_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url, { synchronize_followers: !mention.status.distributable? })
+  def assign_mentions!
+    @current_mentions.each do |mention|
+      mention.save if mention.new_record?
     end
-  end
 
-  def activitypub_json
-    return @activitypub_json if defined?(@activitypub_json)
-    @activitypub_json = Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
+    # If previous mentions are no longer contained in the text, convert them
+    # to silent mentions, since withdrawing access from someone who already
+    # received a notification might be more confusing
+    removed_mentions = @previous_mentions - @current_mentions
+
+    Mention.where(id: removed_mentions.map(&:id)).update_all(silent: true) unless removed_mentions.empty?
   end
 
-  def resolve_account_service
-    ResolveAccountService.new
+  def mention_undeliverable?(mentioned_account)
+    mentioned_account.nil? || (!mentioned_account.local? && !mentioned_account.activitypub?)
   end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 3535b503b..bec95bb1b 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -87,7 +87,7 @@ class RemoveStatusService < BaseService
     # the author and wouldn't normally receive the delete
     # notification - so here, we explicitly send it to them
 
-    status_reach_finder = StatusReachFinder.new(@status)
+    status_reach_finder = StatusReachFinder.new(@status, unsafe: true)
 
     ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes) do |inbox_url|
       [signed_activity_json, @account.id, inbox_url]
diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb
index 09898ca49..17c108461 100644
--- a/app/workers/activitypub/distribution_worker.rb
+++ b/app/workers/activitypub/distribution_worker.rb
@@ -1,54 +1,32 @@
 # frozen_string_literal: true
 
-class ActivityPub::DistributionWorker
-  include Sidekiq::Worker
-  include Payloadable
-
-  sidekiq_options queue: 'push'
-
+class ActivityPub::DistributionWorker < ActivityPub::RawDistributionWorker
+  # Distribute a new status or an edit of a status to all the places
+  # where the status is supposed to go or where it was interacted with
   def perform(status_id)
     @status  = Status.find(status_id)
     @account = @status.account
 
-    return if skip_distribution?
-
-    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, @account.id, inbox_url, { synchronize_followers: !@status.distributable? }]
-    end
-
-    relay! if relayable?
+    distribute!
   rescue ActiveRecord::RecordNotFound
     true
   end
 
-  private
-
-  def skip_distribution?
-    @status.direct_visibility? || @status.limited_visibility?
-  end
-
-  def relayable?
-    @status.public_visibility?
-  end
+  protected
 
   def inboxes
-    # Deliver the status to all followers.
-    # If the status is a reply to another local status, also forward it to that
-    # status' authors' followers.
-    @inboxes ||= if @status.in_reply_to_local_account? && @status.distributable?
-                   @account.followers.or(@status.thread.account.followers).inboxes
-                 else
-                   @account.followers.inboxes
-                 end
+    @inboxes ||= StatusReachFinder.new(@status).inboxes
   end
 
   def payload
-    @payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @account))
+    @payload ||= Oj.dump(serialize_payload(activity, ActivityPub::ActivitySerializer, signer: @account))
+  end
+
+  def activity
+    ActivityPub::ActivityPresenter.from_status(@status)
   end
 
-  def relay!
-    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
-      [payload, @account.id, inbox_url]
-    end
+  def options
+    { synchronize_followers: @status.private_visibility? }
   end
 end
diff --git a/app/workers/activitypub/raw_distribution_worker.rb b/app/workers/activitypub/raw_distribution_worker.rb
index 41e61132f..ac5eda4af 100644
--- a/app/workers/activitypub/raw_distribution_worker.rb
+++ b/app/workers/activitypub/raw_distribution_worker.rb
@@ -2,22 +2,47 @@
 
 class ActivityPub::RawDistributionWorker
   include Sidekiq::Worker
+  include Payloadable
 
   sidekiq_options queue: 'push'
 
+  # Base worker for when you want to queue up a bunch of deliveries of
+  # some payload. In this case, we have already generated JSON and
+  # we are going to distribute it to the account's followers minus
+  # the explicitly provided inboxes
   def perform(json, source_account_id, exclude_inboxes = [])
-    @account = Account.find(source_account_id)
+    @account         = Account.find(source_account_id)
+    @json            = json
+    @exclude_inboxes = exclude_inboxes
 
-    ActivityPub::DeliveryWorker.push_bulk(inboxes - exclude_inboxes) do |inbox_url|
-      [json, @account.id, inbox_url]
-    end
+    distribute!
   rescue ActiveRecord::RecordNotFound
     true
   end
 
-  private
+  protected
+
+  def distribute!
+    return if inboxes.empty?
+
+    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
+      [payload, source_account_id, inbox_url, options]
+    end
+  end
+
+  def payload
+    @json
+  end
+
+  def source_account_id
+    @account.id
+  end
 
   def inboxes
-    @inboxes ||= @account.followers.inboxes
+    @inboxes ||= @account.followers.inboxes - @exclude_inboxes
+  end
+
+  def options
+    nil
   end
 end
diff --git a/app/workers/activitypub/reply_distribution_worker.rb b/app/workers/activitypub/reply_distribution_worker.rb
deleted file mode 100644
index d4d0148ac..000000000
--- a/app/workers/activitypub/reply_distribution_worker.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-# Obsolete but kept around to make sure existing jobs do not fail after upgrade.
-# Should be removed in a subsequent release.
-
-class ActivityPub::ReplyDistributionWorker
-  include Sidekiq::Worker
-  include Payloadable
-
-  sidekiq_options queue: 'push'
-
-  def perform(status_id)
-    @status  = Status.find(status_id)
-    @account = @status.thread&.account
-
-    return unless @account.present? && @status.distributable?
-
-    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, @status.account_id, inbox_url]
-    end
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-
-  private
-
-  def inboxes
-    @inboxes ||= @account.followers.inboxes
-  end
-
-  def payload
-    @payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
-  end
-end
diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb
index 3a207f071..81fde63b8 100644
--- a/app/workers/activitypub/update_distribution_worker.rb
+++ b/app/workers/activitypub/update_distribution_worker.rb
@@ -1,33 +1,24 @@
 # frozen_string_literal: true
 
-class ActivityPub::UpdateDistributionWorker
-  include Sidekiq::Worker
-  include Payloadable
-
-  sidekiq_options queue: 'push'
-
+class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker
+  # Distribute an profile update to servers that might have a copy
+  # of the account in question
   def perform(account_id, options = {})
     @options = options.with_indifferent_access
     @account = Account.find(account_id)
 
-    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [signed_payload, @account.id, inbox_url]
-    end
-
-    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
-      [signed_payload, @account.id, inbox_url]
-    end
+    distribute!
   rescue ActiveRecord::RecordNotFound
     true
   end
 
-  private
+  protected
 
   def inboxes
-    @inboxes ||= @account.followers.inboxes
+    @inboxes ||= AccountReachFinder.new(@account).inboxes
   end
 
-  def signed_payload
-    @signed_payload ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account, sign_with: @options[:sign_with]))
+  def payload
+    @payload ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account, sign_with: @options[:sign_with]))
   end
 end
diff --git a/app/workers/distribution_worker.rb b/app/workers/distribution_worker.rb
index e85cd7e95..770325ccf 100644
--- a/app/workers/distribution_worker.rb
+++ b/app/workers/distribution_worker.rb
@@ -3,10 +3,10 @@
 class DistributionWorker
   include Sidekiq::Worker
 
-  def perform(status_id)
+  def perform(status_id, options = {})
     RedisLock.acquire(redis: Redis.current, key: "distribute:#{status_id}", autorelease: 5.minutes.seconds) do |lock|
       if lock.acquired?
-        FanOutOnWriteService.new.call(Status.find(status_id))
+        FanOutOnWriteService.new.call(Status.find(status_id), **options.symbolize_keys)
       else
         raise Mastodon::RaceConditionError
       end
diff --git a/app/workers/feed_insert_worker.rb b/app/workers/feed_insert_worker.rb
index b70c7e389..0122be95d 100644
--- a/app/workers/feed_insert_worker.rb
+++ b/app/workers/feed_insert_worker.rb
@@ -3,9 +3,10 @@
 class FeedInsertWorker
   include Sidekiq::Worker
 
-  def perform(status_id, id, type = :home)
-    @type     = type.to_sym
-    @status   = Status.find(status_id)
+  def perform(status_id, id, type = :home, options = {})
+    @type      = type.to_sym
+    @status    = Status.find(status_id)
+    @options   = options.symbolize_keys
 
     case @type
     when :home
@@ -23,10 +24,12 @@ class FeedInsertWorker
   private
 
   def check_and_insert
-    return if feed_filtered?
-
-    perform_push
-    perform_notify if notify?
+    if feed_filtered?
+      perform_unpush if update?
+    else
+      perform_push
+      perform_notify if notify?
+    end
   end
 
   def feed_filtered?
@@ -47,13 +50,26 @@ class FeedInsertWorker
   def perform_push
     case @type
     when :home
-      FeedManager.instance.push_to_home(@follower, @status)
+      FeedManager.instance.push_to_home(@follower, @status, update: update?)
+    when :list
+      FeedManager.instance.push_to_list(@list, @status, update: update?)
+    end
+  end
+
+  def perform_unpush
+    case @type
+    when :home
+      FeedManager.instance.unpush_from_home(@follower, @status, update: true)
     when :list
-      FeedManager.instance.push_to_list(@list, @status)
+      FeedManager.instance.unpush_from_list(@list, @status, update: true)
     end
   end
 
   def perform_notify
     NotifyService.new.call(@follower, :status, @status)
   end
+
+  def update?
+    @options[:update]
+  end
 end
diff --git a/app/workers/local_notification_worker.rb b/app/workers/local_notification_worker.rb
index 6b08ca6fc..a22e2834d 100644
--- a/app/workers/local_notification_worker.rb
+++ b/app/workers/local_notification_worker.rb
@@ -12,6 +12,8 @@ class LocalNotificationWorker
       activity = activity_class_name.constantize.find(activity_id)
     end
 
+    return if Notification.where(account: receiver, activity: activity).any?
+
     NotifyService.new.call(receiver, type || activity_class_name.underscore, activity)
   rescue ActiveRecord::RecordNotFound
     true
diff --git a/app/workers/poll_expiration_notify_worker.rb b/app/workers/poll_expiration_notify_worker.rb
index f0191d479..7613ed5f1 100644
--- a/app/workers/poll_expiration_notify_worker.rb
+++ b/app/workers/poll_expiration_notify_worker.rb
@@ -6,19 +6,44 @@ class PollExpirationNotifyWorker
   sidekiq_options lock: :until_executed
 
   def perform(poll_id)
-    poll = Poll.find(poll_id)
+    @poll = Poll.find(poll_id)
 
-    # Notify poll owner and remote voters
-    if poll.local?
-      ActivityPub::DistributePollUpdateWorker.perform_async(poll.status.id)
-      NotifyService.new.call(poll.account, :poll, poll)
-    end
+    return if does_not_expire?
+    requeue! && return if not_due_yet?
 
-    # Notify local voters
-    poll.votes.includes(:account).group(:account_id).select(:account_id).map(&:account).select(&:local?).each do |account|
-      NotifyService.new.call(account, :poll, poll)
-    end
+    notify_remote_voters_and_owner! if @poll.local?
+    notify_local_voters!
   rescue ActiveRecord::RecordNotFound
     true
   end
+
+  def self.remove_from_scheduled(poll_id)
+    queue = Sidekiq::ScheduledSet.new
+    queue.select { |scheduled| scheduled.klass == name && scheduled.args[0] == poll_id }.map(&:delete)
+  end
+
+  private
+
+  def does_not_expire?
+    @poll.expires_at.nil?
+  end
+
+  def not_due_yet?
+    @poll.expires_at.present? && !@poll.expired?
+  end
+
+  def requeue!
+    PollExpirationNotifyWorker.perform_at(@poll.expires_at + 5.minutes, @poll.id)
+  end
+
+  def notify_remote_voters_and_owner!
+    ActivityPub::DistributePollUpdateWorker.perform_async(@poll.status.id)
+    NotifyService.new.call(@poll.account, :poll, @poll)
+  end
+
+  def notify_local_voters!
+    @poll.voters.merge(Account.local).find_each do |account|
+      NotifyService.new.call(account, :poll, @poll)
+    end
+  end
 end
diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb
index d76d73d96..ae444cfde 100644
--- a/app/workers/push_update_worker.rb
+++ b/app/workers/push_update_worker.rb
@@ -2,15 +2,38 @@
 
 class PushUpdateWorker
   include Sidekiq::Worker
+  include Redisable
 
-  def perform(account_id, status_id, timeline_id = nil)
-    account     = Account.find(account_id)
-    status      = Status.find(status_id)
-    message     = InlineRenderer.render(status, account, :status)
-    timeline_id = "timeline:#{account.id}" if timeline_id.nil?
+  def perform(account_id, status_id, timeline_id = nil, options = {})
+    @account     = Account.find(account_id)
+    @status      = Status.find(status_id)
+    @timeline_id = timeline_id || "timeline:#{account.id}"
+    @options     = options.symbolize_keys
 
-    Redis.current.publish(timeline_id, Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
+    publish!
   rescue ActiveRecord::RecordNotFound
     true
   end
+
+  private
+
+  def payload
+    InlineRenderer.render(@status, @account, :status)
+  end
+
+  def message
+    Oj.dump(
+      event: update? ? :'status.update' : :update,
+      payload: payload,
+      queued_at: (Time.now.to_f * 1000.0).to_i
+    )
+  end
+
+  def publish!
+    redis.publish(@timeline_id, message)
+  end
+
+  def update?
+    @options[:update]
+  end
 end
diff --git a/config/routes.rb b/config/routes.rb
index 41ba45379..121587819 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -350,6 +350,9 @@ Rails.application.routes.draw do
 
           resource :pin, only: :create
           post :unpin, to: 'pins#destroy'
+
+          resource :history, only: :show
+          resource :source, only: :show
         end
 
         member do
diff --git a/db/migrate/20210904215403_add_edited_at_to_statuses.rb b/db/migrate/20210904215403_add_edited_at_to_statuses.rb
new file mode 100644
index 000000000..216ad8e13
--- /dev/null
+++ b/db/migrate/20210904215403_add_edited_at_to_statuses.rb
@@ -0,0 +1,5 @@
+class AddEditedAtToStatuses < ActiveRecord::Migration[6.1]
+  def change
+    add_column :statuses, :edited_at, :datetime
+  end
+end
diff --git a/db/migrate/20210908220918_create_status_edits.rb b/db/migrate/20210908220918_create_status_edits.rb
new file mode 100644
index 000000000..6c90149d0
--- /dev/null
+++ b/db/migrate/20210908220918_create_status_edits.rb
@@ -0,0 +1,13 @@
+class CreateStatusEdits < ActiveRecord::Migration[6.1]
+  def change
+    create_table :status_edits do |t|
+      t.belongs_to :status, null: false, foreign_key: { on_delete: :cascade }
+      t.belongs_to :account, null: true, foreign_key: { on_delete: :nullify }
+      t.text :text, null: false, default: ''
+      t.text :spoiler_text, null: false, default: ''
+      t.boolean :media_attachments_changed, null: false, default: false
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ed615a1ee..4e0f76dcd 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -816,6 +816,18 @@ ActiveRecord::Schema.define(version: 2022_01_16_202951) do
     t.index ["var"], name: "index_site_uploads_on_var", unique: true
   end
 
+  create_table "status_edits", force: :cascade do |t|
+    t.bigint "status_id", null: false
+    t.bigint "account_id"
+    t.text "text", default: "", null: false
+    t.text "spoiler_text", default: "", null: false
+    t.boolean "media_attachments_changed", default: false, null: false
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["account_id"], name: "index_status_edits_on_account_id"
+    t.index ["status_id"], name: "index_status_edits_on_status_id"
+  end
+
   create_table "status_pins", force: :cascade do |t|
     t.bigint "account_id", null: false
     t.bigint "status_id", null: false
@@ -854,6 +866,7 @@ ActiveRecord::Schema.define(version: 2022_01_16_202951) do
     t.bigint "in_reply_to_account_id"
     t.bigint "poll_id"
     t.datetime "deleted_at"
+    t.datetime "edited_at"
     t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
     t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
     t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
@@ -1081,6 +1094,8 @@ ActiveRecord::Schema.define(version: 2022_01_16_202951) do
   add_foreign_key "scheduled_statuses", "accounts", on_delete: :cascade
   add_foreign_key "session_activations", "oauth_access_tokens", column: "access_token_id", name: "fk_957e5bda89", on_delete: :cascade
   add_foreign_key "session_activations", "users", name: "fk_e5fda67334", on_delete: :cascade
+  add_foreign_key "status_edits", "accounts", on_delete: :nullify
+  add_foreign_key "status_edits", "statuses", on_delete: :cascade
   add_foreign_key "status_pins", "accounts", name: "fk_d4cb435b62", on_delete: :cascade
   add_foreign_key "status_pins", "statuses", on_delete: :cascade
   add_foreign_key "status_stats", "statuses", on_delete: :cascade
diff --git a/spec/controllers/api/v1/statuses/histories_controller_spec.rb b/spec/controllers/api/v1/statuses/histories_controller_spec.rb
new file mode 100644
index 000000000..8d9d6a359
--- /dev/null
+++ b/spec/controllers/api/v1/statuses/histories_controller_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Api::V1::Statuses::HistoriesController do
+  render_views
+
+  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let(:app)   { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses', application: app) }
+
+  context 'with an oauth token' do
+    before do
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    describe 'GET #show' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        get :show, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/statuses/sources_controller_spec.rb b/spec/controllers/api/v1/statuses/sources_controller_spec.rb
new file mode 100644
index 000000000..293c90ec9
--- /dev/null
+++ b/spec/controllers/api/v1/statuses/sources_controller_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Api::V1::Statuses::SourcesController do
+  render_views
+
+  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let(:app)   { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses', application: app) }
+
+  context 'with an oauth token' do
+    before do
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    describe 'GET #show' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        get :show, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/preview_card_fabricator.rb b/spec/fabricators/preview_card_fabricator.rb
new file mode 100644
index 000000000..f119c117d
--- /dev/null
+++ b/spec/fabricators/preview_card_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:preview_card) do
+  url { Faker::Internet.url }
+  title { Faker::Lorem.sentence }
+  description { Faker::Lorem.paragraph }
+  type 'link'
+end
diff --git a/spec/fabricators/status_edit_fabricator.rb b/spec/fabricators/status_edit_fabricator.rb
new file mode 100644
index 000000000..21b793747
--- /dev/null
+++ b/spec/fabricators/status_edit_fabricator.rb
@@ -0,0 +1,7 @@
+Fabricator(:status_edit) do
+  status                    nil
+  account                   nil
+  text                      "MyText"
+  spoiler_text              "MyText"
+  media_attachments_changed false
+end
\ No newline at end of file
diff --git a/spec/lib/status_reach_finder_spec.rb b/spec/lib/status_reach_finder_spec.rb
new file mode 100644
index 000000000..f0c22b165
--- /dev/null
+++ b/spec/lib/status_reach_finder_spec.rb
@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe StatusReachFinder do
+  describe '#inboxes' do
+    context 'for a local status' do
+      let(:parent_status) { nil }
+      let(:visibility) { :public }
+      let(:alice) { Fabricate(:account, username: 'alice') }
+      let(:status) { Fabricate(:status, account: alice, thread: parent_status, visibility: visibility) }
+
+      subject { described_class.new(status) }
+
+      context 'when it contains mentions of remote accounts' do
+        let(:bob) { Fabricate(:account, username: 'bob', domain: 'foo.bar', protocol: :activitypub, inbox_url: 'https://foo.bar/inbox') }
+
+        before do
+          status.mentions.create!(account: bob)
+        end
+
+        it 'includes the inbox of the mentioned account' do
+          expect(subject.inboxes).to include 'https://foo.bar/inbox'
+        end
+      end
+
+      context 'when it has been reblogged by a remote account' do
+        let(:bob) { Fabricate(:account, username: 'bob', domain: 'foo.bar', protocol: :activitypub, inbox_url: 'https://foo.bar/inbox') }
+
+        before do
+          bob.statuses.create!(reblog: status)
+        end
+
+        it 'includes the inbox of the reblogger' do
+          expect(subject.inboxes).to include 'https://foo.bar/inbox'
+        end
+
+        context 'when status is not public' do
+          let(:visibility) { :private }
+
+          it 'does not include the inbox of the reblogger' do
+            expect(subject.inboxes).to_not include 'https://foo.bar/inbox'
+          end
+        end
+      end
+
+      context 'when it has been favourited by a remote account' do
+        let(:bob) { Fabricate(:account, username: 'bob', domain: 'foo.bar', protocol: :activitypub, inbox_url: 'https://foo.bar/inbox') }
+
+        before do
+          bob.favourites.create!(status: status)
+        end
+
+        it 'includes the inbox of the favouriter' do
+          expect(subject.inboxes).to include 'https://foo.bar/inbox'
+        end
+
+        context 'when status is not public' do
+          let(:visibility) { :private }
+
+          it 'does not include the inbox of the favouriter' do
+            expect(subject.inboxes).to_not include 'https://foo.bar/inbox'
+          end
+        end
+      end
+
+      context 'when it has been replied to by a remote account' do
+        let(:bob) { Fabricate(:account, username: 'bob', domain: 'foo.bar', protocol: :activitypub, inbox_url: 'https://foo.bar/inbox') }
+
+        before do
+          bob.statuses.create!(thread: status, text: 'Hoge')
+        end
+
+        context do
+          it 'includes the inbox of the replier' do
+            expect(subject.inboxes).to include 'https://foo.bar/inbox'
+          end
+        end
+
+        context 'when status is not public' do
+          let(:visibility) { :private }
+
+          it 'does not include the inbox of the replier' do
+            expect(subject.inboxes).to_not include 'https://foo.bar/inbox'
+          end
+        end
+      end
+
+      context 'when it is a reply to a remote account' do
+        let(:bob) { Fabricate(:account, username: 'bob', domain: 'foo.bar', protocol: :activitypub, inbox_url: 'https://foo.bar/inbox') }
+        let(:parent_status) { Fabricate(:status, account: bob) }
+
+        context do
+          it 'includes the inbox of the replied-to account' do
+            expect(subject.inboxes).to include 'https://foo.bar/inbox'
+          end
+        end
+
+        context 'when status is not public and replied-to account is not mentioned' do
+          let(:visibility) { :private }
+
+          it 'does not include the inbox of the replied-to account' do
+            expect(subject.inboxes).to_not include 'https://foo.bar/inbox'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/models/status_edit_spec.rb b/spec/models/status_edit_spec.rb
new file mode 100644
index 000000000..2ecafef73
--- /dev/null
+++ b/spec/models/status_edit_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe StatusEdit, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/spec/services/activitypub/fetch_remote_status_service_spec.rb b/spec/services/activitypub/fetch_remote_status_service_spec.rb
index ceba5f210..94574aa7f 100644
--- a/spec/services/activitypub/fetch_remote_status_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_status_service_spec.rb
@@ -67,7 +67,7 @@ RSpec.describe ActivityPub::FetchRemoteStatusService, type: :service do
 
         expect(status).to_not be_nil
         expect(status.url).to eq "https://#{valid_domain}/watch?v=12345"
-        expect(strip_tags(status.text)).to eq "Nyan Cat 10 hours remix https://#{valid_domain}/watch?v=12345"
+        expect(strip_tags(status.text)).to eq "Nyan Cat 10 hours remixhttps://#{valid_domain}/watch?v=12345"
       end
     end
 
@@ -100,7 +100,7 @@ RSpec.describe ActivityPub::FetchRemoteStatusService, type: :service do
 
         expect(status).to_not be_nil
         expect(status.url).to eq "https://#{valid_domain}/watch?v=12345"
-        expect(strip_tags(status.text)).to eq "Nyan Cat 10 hours remix https://#{valid_domain}/watch?v=12345"
+        expect(strip_tags(status.text)).to eq "Nyan Cat 10 hours remixhttps://#{valid_domain}/watch?v=12345"
       end
     end
 
@@ -120,7 +120,7 @@ RSpec.describe ActivityPub::FetchRemoteStatusService, type: :service do
 
         expect(status).to_not be_nil
         expect(status.url).to eq "https://#{valid_domain}/@foo/1234"
-        expect(strip_tags(status.text)).to eq "Let's change the world https://#{valid_domain}/@foo/1234"
+        expect(strip_tags(status.text)).to eq "Let's change the worldhttps://#{valid_domain}/@foo/1234"
       end
     end
 
diff --git a/spec/services/fan_out_on_write_service_spec.rb b/spec/services/fan_out_on_write_service_spec.rb
index 538dc2592..4ce110e45 100644
--- a/spec/services/fan_out_on_write_service_spec.rb
+++ b/spec/services/fan_out_on_write_service_spec.rb
@@ -1,37 +1,112 @@
 require 'rails_helper'
 
 RSpec.describe FanOutOnWriteService, type: :service do
-  let(:author)   { Fabricate(:account, username: 'tom') }
-  let(:status)   { Fabricate(:status, text: 'Hello @alice #test', account: author) }
-  let(:alice)    { Fabricate(:user, account: Fabricate(:account, username: 'alice')).account }
-  let(:follower) { Fabricate(:account, username: 'bob') }
+  let(:last_active_at) { Time.now.utc }
 
-  subject { FanOutOnWriteService.new }
+  let!(:alice) { Fabricate(:user, current_sign_in_at: last_active_at, account: Fabricate(:account, username: 'alice')).account }
+  let!(:bob)   { Fabricate(:user, current_sign_in_at: last_active_at, account: Fabricate(:account, username: 'bob')).account }
+  let!(:tom)   { Fabricate(:user, current_sign_in_at: last_active_at, account: Fabricate(:account, username: 'tom')).account }
+
+  subject { described_class.new }
+
+  let(:status) { Fabricate(:status, account: alice, visibility: visibility, text: 'Hello @bob #hoge') }
 
   before do
-    alice
-    follower.follow!(author)
+    bob.follow!(alice)
+    tom.follow!(alice)
 
     ProcessMentionsService.new.call(status)
     ProcessHashtagsService.new.call(status)
 
+    allow(Redis.current).to receive(:publish)
+
     subject.call(status)
   end
 
-  it 'delivers status to home timeline' do
-    expect(HomeFeed.new(author).get(10).map(&:id)).to include status.id
+  def home_feed_of(account)
+    HomeFeed.new(account).get(10).map(&:id)
+  end
+
+  context 'when status is public' do
+    let(:visibility) { 'public' }
+
+    it 'is added to the home feed of its author' do
+      expect(home_feed_of(alice)).to include status.id
+    end
+
+    it 'is added to the home feed of a follower' do
+      expect(home_feed_of(bob)).to include status.id
+      expect(home_feed_of(tom)).to include status.id
+    end
+
+    it 'is broadcast to the hashtag stream' do
+      expect(Redis.current).to have_received(:publish).with('timeline:hashtag:hoge', anything)
+      expect(Redis.current).to have_received(:publish).with('timeline:hashtag:hoge:local', anything)
+    end
+
+    it 'is broadcast to the public stream' do
+      expect(Redis.current).to have_received(:publish).with('timeline:public', anything)
+      expect(Redis.current).to have_received(:publish).with('timeline:public:local', anything)
+    end
   end
 
-  it 'delivers status to local followers' do
-    pending 'some sort of problem in test environment causes this to sometimes fail'
-    expect(HomeFeed.new(follower).get(10).map(&:id)).to include status.id
+  context 'when status is limited' do
+    let(:visibility) { 'limited' }
+
+    it 'is added to the home feed of its author' do
+      expect(home_feed_of(alice)).to include status.id
+    end
+
+    it 'is added to the home feed of the mentioned follower' do
+      expect(home_feed_of(bob)).to include status.id
+    end
+
+    it 'is not added to the home feed of the other follower' do
+      expect(home_feed_of(tom)).to_not include status.id
+    end
+
+    it 'is not broadcast publicly' do
+      expect(Redis.current).to_not have_received(:publish).with('timeline:hashtag:hoge', anything)
+      expect(Redis.current).to_not have_received(:publish).with('timeline:public', anything)
+    end
   end
 
-  it 'delivers status to hashtag' do
-    expect(TagFeed.new(Tag.find_by(name: 'test'), alice).get(20).map(&:id)).to include status.id
+  context 'when status is private' do
+    let(:visibility) { 'private' }
+
+    it 'is added to the home feed of its author' do
+      expect(home_feed_of(alice)).to include status.id
+    end
+
+    it 'is added to the home feed of a follower' do
+      expect(home_feed_of(bob)).to include status.id
+      expect(home_feed_of(tom)).to include status.id
+    end
+
+    it 'is not broadcast publicly' do
+      expect(Redis.current).to_not have_received(:publish).with('timeline:hashtag:hoge', anything)
+      expect(Redis.current).to_not have_received(:publish).with('timeline:public', anything)
+    end
   end
 
-  it 'delivers status to public timeline' do
-    expect(PublicFeed.new(alice).get(20).map(&:id)).to include status.id
+  context 'when status is direct' do
+    let(:visibility) { 'direct' }
+
+    it 'is added to the home feed of its author' do
+      expect(home_feed_of(alice)).to include status.id
+    end
+
+    it 'is added to the home feed of the mentioned follower' do
+      expect(home_feed_of(bob)).to include status.id
+    end
+
+    it 'is not added to the home feed of the other follower' do
+      expect(home_feed_of(tom)).to_not include status.id
+    end
+
+    it 'is not broadcast publicly' do
+      expect(Redis.current).to_not have_received(:publish).with('timeline:hashtag:hoge', anything)
+      expect(Redis.current).to_not have_received(:publish).with('timeline:public', anything)
+    end
   end
 end
diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb
index d74e8dc62..89b265e9a 100644
--- a/spec/services/process_mentions_service_spec.rb
+++ b/spec/services/process_mentions_service_spec.rb
@@ -9,75 +9,55 @@ RSpec.describe ProcessMentionsService, type: :service do
 
   context 'ActivityPub' do
     context do
-      let(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox') }
+      let!(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox') }
 
       before do
-        stub_request(:post, remote_user.inbox_url)
         subject.call(status)
       end
 
       it 'creates a mention' do
         expect(remote_user.mentions.where(status: status).count).to eq 1
       end
-
-      it 'sends activity to the inbox' do
-        expect(a_request(:post, remote_user.inbox_url)).to have_been_made.once
-      end
     end
 
     context 'with an IDN domain' do
-      let(:remote_user) { Fabricate(:account, username: 'sneak', protocol: :activitypub, domain: 'xn--hresiar-mxa.ch', inbox_url: 'http://example.com/inbox') }
-      let(:status) { Fabricate(:status, account: account, text: "Hello @sneak@hæresiar.ch") }
+      let!(:remote_user) { Fabricate(:account, username: 'sneak', protocol: :activitypub, domain: 'xn--hresiar-mxa.ch', inbox_url: 'http://example.com/inbox') }
+      let!(:status) { Fabricate(:status, account: account, text: "Hello @sneak@hæresiar.ch") }
 
       before do
-        stub_request(:post, remote_user.inbox_url)
         subject.call(status)
       end
 
       it 'creates a mention' do
         expect(remote_user.mentions.where(status: status).count).to eq 1
       end
-
-      it 'sends activity to the inbox' do
-        expect(a_request(:post, remote_user.inbox_url)).to have_been_made.once
-      end
     end
 
     context 'with an IDN TLD' do
-      let(:remote_user) { Fabricate(:account, username: 'foo', protocol: :activitypub, domain: 'xn--y9a3aq.xn--y9a3aq', inbox_url: 'http://example.com/inbox') }
-      let(:status) { Fabricate(:status, account: account, text: "Hello @foo@հայ.հայ") }
+      let!(:remote_user) { Fabricate(:account, username: 'foo', protocol: :activitypub, domain: 'xn--y9a3aq.xn--y9a3aq', inbox_url: 'http://example.com/inbox') }
+      let!(:status) { Fabricate(:status, account: account, text: "Hello @foo@հայ.հայ") }
 
       before do
-        stub_request(:post, remote_user.inbox_url)
         subject.call(status)
       end
 
       it 'creates a mention' do
         expect(remote_user.mentions.where(status: status).count).to eq 1
       end
-
-      it 'sends activity to the inbox' do
-        expect(a_request(:post, remote_user.inbox_url)).to have_been_made.once
-      end
     end
   end
 
   context 'Temporarily-unreachable ActivityPub user' do
-    let(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox', last_webfingered_at: nil) }
+    let!(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox', last_webfingered_at: nil) }
 
     before do
       stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404)
       stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:remote_user@example.com").to_return(status: 500)
-      stub_request(:post, remote_user.inbox_url)
       subject.call(status)
     end
 
     it 'creates a mention' do
       expect(remote_user.mentions.where(status: status).count).to eq 1
     end
-
-    it 'sends activity to the inbox' do
-      expect(a_request(:post, remote_user.inbox_url)).to have_been_made.once
-    end
   end
 end
diff --git a/spec/workers/activitypub/distribution_worker_spec.rb b/spec/workers/activitypub/distribution_worker_spec.rb
index 368ca025a..c017b4da1 100644
--- a/spec/workers/activitypub/distribution_worker_spec.rb
+++ b/spec/workers/activitypub/distribution_worker_spec.rb
@@ -35,13 +35,16 @@ describe ActivityPub::DistributionWorker do
     end
 
     context 'with direct status' do
+      let(:mentioned_account) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/inbox')}
+
       before do
         status.update(visibility: :direct)
+        status.mentions.create!(account: mentioned_account)
       end
 
-      it 'does nothing' do
+      it 'delivers to mentioned accounts' do
         subject.perform(status.id)
-        expect(ActivityPub::DeliveryWorker).to_not have_received(:push_bulk)
+        expect(ActivityPub::DeliveryWorker).to have_received(:push_bulk).with(['https://foo.bar/inbox'])
       end
     end
   end
diff --git a/spec/workers/feed_insert_worker_spec.rb b/spec/workers/feed_insert_worker_spec.rb
index 3509f1f50..fb34970fc 100644
--- a/spec/workers/feed_insert_worker_spec.rb
+++ b/spec/workers/feed_insert_worker_spec.rb
@@ -45,7 +45,7 @@ describe FeedInsertWorker do
         result = subject.perform(status.id, follower.id)
 
         expect(result).to be_nil
-        expect(instance).to have_received(:push_to_home).with(follower, status)
+        expect(instance).to have_received(:push_to_home).with(follower, status, update: nil)
       end
     end
   end
-- 
cgit 


From d412a8d1f239aa93a92f420127cb3183a8bb6449 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 19 Jan 2022 22:50:01 +0100
Subject: Fix error when processing poll updates (#17333)

Regression from #16697
---
 app/services/activitypub/process_status_update_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb
index e3e9b9d6a..ed10a0063 100644
--- a/app/services/activitypub/process_status_update_service.rb
+++ b/app/services/activitypub/process_status_update_service.rb
@@ -78,7 +78,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     MediaAttachment.where(id: removed_media_attachments.map(&:id)).update_all(status_id: nil)
     MediaAttachment.where(id: added_media_attachments.map(&:id)).update_all(status_id: @status.id)
 
-    @media_attachments_changed = true if removed_media_attachments.positive? || added_media_attachments.positive?
+    @media_attachments_changed = true if removed_media_attachments.any? || added_media_attachments.any?
   end
 
   def update_poll!
-- 
cgit 


From 6505b39e5d28abf938bd5f593eda2988d322887b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 26 Jan 2022 18:05:39 +0100
Subject: Fix poll updates being saved as status edits (#17373)

Fix #17344
---
 .../activitypub/process_status_update_service.rb   |  22 +++--
 spec/lib/activitypub/activity/update_spec.rb       | 110 +++++++++++++++------
 2 files changed, 97 insertions(+), 35 deletions(-)

(limited to 'app/services')

diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb
index ed10a0063..e22ea1ab6 100644
--- a/app/services/activitypub/process_status_update_service.rb
+++ b/app/services/activitypub/process_status_update_service.rb
@@ -10,6 +10,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     @status                    = status
     @account                   = status.account
     @media_attachments_changed = false
+    @poll_changed              = false
 
     # Only native types can be updated at the moment
     return if !expected_type? || already_updated_more_recently?
@@ -27,6 +28,9 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
         end
 
         queue_poll_notifications!
+
+        next unless significant_changes?
+
         reset_preview_card!
         broadcast_updates!
       else
@@ -92,7 +96,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
       # If for some reasons the options were changed, it invalidates all previous
       # votes, so we need to remove them
       if poll_parser.significantly_changes?(poll)
-        @media_attachments_changed = true
+        @poll_changed = true
         poll.votes.delete_all unless poll.new_record?
       end
 
@@ -107,7 +111,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
       @status.poll_id = poll.id
     elsif previous_poll.present?
       previous_poll.destroy!
-      @media_attachments_changed = true
+      @poll_changed = true
       @status.poll_id = nil
     end
   end
@@ -117,7 +121,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     @status.spoiler_text = @status_parser.spoiler_text || ''
     @status.sensitive    = @account.sensitized? || @status_parser.sensitive || false
     @status.language     = @status_parser.language || detected_language
-    @status.edited_at    = @status_parser.edited_at || Time.now.utc
+    @status.edited_at    = @status_parser.edited_at || Time.now.utc if significant_changes?
 
     @status.save!
   end
@@ -227,12 +231,12 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   end
 
   def create_edit!
-    return unless @status.text_previously_changed? || @status.spoiler_text_previously_changed? || @media_attachments_changed
+    return unless significant_changes?
 
     @status_edit = @status.edits.create(
       text: @status.text,
       spoiler_text: @status.spoiler_text,
-      media_attachments_changed: @media_attachments_changed,
+      media_attachments_changed: @media_attachments_changed || @poll_changed,
       account_id: @account.id,
       created_at: @status.edited_at
     )
@@ -248,13 +252,17 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type)
   end
 
+  def significant_changes?
+    @status.text_changed? || @status.text_previously_changed? || @status.spoiler_text_changed? || @status.spoiler_text_previously_changed? || @media_attachments_changed || @poll_changed
+  end
+
   def already_updated_more_recently?
     @status.edited_at.present? && @status_parser.edited_at.present? && @status.edited_at > @status_parser.edited_at
   end
 
   def reset_preview_card!
-    @status.preview_cards.clear if @status.text_previously_changed? || @status.spoiler_text.present?
-    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id) if @status.spoiler_text.blank?
+    @status.preview_cards.clear
+    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id)
   end
 
   def broadcast_updates!
diff --git a/spec/lib/activitypub/activity/update_spec.rb b/spec/lib/activitypub/activity/update_spec.rb
index 1c9bcf43b..4cd853af2 100644
--- a/spec/lib/activitypub/activity/update_spec.rb
+++ b/spec/lib/activitypub/activity/update_spec.rb
@@ -4,43 +4,97 @@ RSpec.describe ActivityPub::Activity::Update do
   let!(:sender) { Fabricate(:account) }
 
   before do
-    stub_request(:get, actor_json[:outbox]).to_return(status: 404)
-    stub_request(:get, actor_json[:followers]).to_return(status: 404)
-    stub_request(:get, actor_json[:following]).to_return(status: 404)
-    stub_request(:get, actor_json[:featured]).to_return(status: 404)
-
     sender.update!(uri: ActivityPub::TagManager.instance.uri_for(sender))
   end
 
-  let(:modified_sender) do
-    sender.tap do |modified_sender|
-      modified_sender.display_name = 'Totally modified now'
-    end
-  end
+  subject { described_class.new(json, sender) }
 
-  let(:actor_json) do
-    ActiveModelSerializers::SerializableResource.new(modified_sender, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter).as_json
-  end
+  describe '#perform' do
+    context 'with an Actor object' do
+      let(:modified_sender) do
+        sender.tap do |modified_sender|
+          modified_sender.display_name = 'Totally modified now'
+        end
+      end
 
-  let(:json) do
-    {
-      '@context': 'https://www.w3.org/ns/activitystreams',
-      id: 'foo',
-      type: 'Update',
-      actor: ActivityPub::TagManager.instance.uri_for(sender),
-      object: actor_json,
-    }.with_indifferent_access
-  end
+      let(:actor_json) do
+        ActiveModelSerializers::SerializableResource.new(modified_sender, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter).as_json
+      end
 
-  describe '#perform' do
-    subject { described_class.new(json, sender) }
+      let(:json) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: 'foo',
+          type: 'Update',
+          actor: ActivityPub::TagManager.instance.uri_for(sender),
+          object: actor_json,
+        }.with_indifferent_access
+      end
 
-    before do
-      subject.perform
+      before do
+        stub_request(:get, actor_json[:outbox]).to_return(status: 404)
+        stub_request(:get, actor_json[:followers]).to_return(status: 404)
+        stub_request(:get, actor_json[:following]).to_return(status: 404)
+        stub_request(:get, actor_json[:featured]).to_return(status: 404)
+
+        subject.perform
+      end
+
+      it 'updates profile' do
+        expect(sender.reload.display_name).to eq 'Totally modified now'
+      end
     end
 
-    it 'updates profile' do
-      expect(sender.reload.display_name).to eq 'Totally modified now'
+    context 'with a Question object' do
+      let!(:at_time) { Time.now.utc }
+      let!(:status) { Fabricate(:status, account: sender, poll: Poll.new(account: sender, options: %w(Bar Baz), cached_tallies: [0, 0], expires_at: at_time + 5.days)) }
+
+      let(:json) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: 'foo',
+          type: 'Update',
+          actor: ActivityPub::TagManager.instance.uri_for(sender),
+          object: {
+            type: 'Question',
+            id: ActivityPub::TagManager.instance.uri_for(status),
+            content: 'Foo',
+            endTime: (at_time + 5.days).iso8601,
+            oneOf: [
+              {
+                type: 'Note',
+                name: 'Bar',
+                replies: {
+                  type: 'Collection',
+                  totalItems: 0,
+                },
+              },
+
+              {
+                type: 'Note',
+                name: 'Baz',
+                replies: {
+                  type: 'Collection',
+                  totalItems: 12,
+                },
+              },
+            ],
+          },
+        }.with_indifferent_access
+      end
+
+      before do
+        status.update!(uri: ActivityPub::TagManager.instance.uri_for(status))
+        subject.perform
+      end
+
+      it 'updates poll numbers' do
+        expect(status.preloadable_poll.cached_tallies).to eq [0, 12]
+      end
+
+      it 'does not set status as edited' do
+        expect(status.edited_at).to be_nil
+      end
     end
   end
 end
-- 
cgit 


From 166cc5b89d60f8e2e1f748df86ff8a9003a4876e Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 26 Jan 2022 20:53:50 +0100
Subject: Fix local distribution of edited statuses (#17380)

Because `FanOutOnWriteService#update?` was broken, edits were considered as new
toots and a regular `update` payload was sent.
---
 app/services/fan_out_on_write_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index f62f78a79..3a2e82b6d 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -119,7 +119,7 @@ class FanOutOnWriteService < BaseService
   end
 
   def update?
-    @is_update
+    @options[:update]
   end
 
   def broadcastable?
-- 
cgit 


From 03d59340da3ffc380d7b27169bdc887140d88bee Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 28 Jan 2022 00:43:56 +0100
Subject: Fix Sidekiq warnings about JSON serialization (#17381)

* Fix Sidekiq warnings about JSON serialization

This occurs on every symbol argument we pass, and every symbol key in hashes,
because Sidekiq expects strings instead.

See https://github.com/mperham/sidekiq/pull/5071

We do not need to change how workers parse their arguments because this has
not changed and we were already converting to symbols adequately or using
`with_indifferent_access`.

* Set Sidekiq to raise on unsafe arguments in test mode

In order to more easily catch issues that would produce warnings in production
code.
---
 app/controllers/api/v1/statuses_controller.rb             | 2 +-
 app/lib/activitypub/activity/create.rb                    | 4 ++--
 app/lib/feed_manager.rb                                   | 4 ++--
 app/models/admin/status_batch_action.rb                   | 2 +-
 app/models/form/account_batch.rb                          | 2 +-
 app/services/account_statuses_cleanup_service.rb          | 2 +-
 app/services/activitypub/process_status_update_service.rb | 2 +-
 app/services/fan_out_on_write_service.rb                  | 8 ++++----
 app/services/follow_service.rb                            | 4 ++--
 app/services/import_service.rb                            | 4 ++--
 app/services/reblog_service.rb                            | 2 +-
 app/services/resolve_account_service.rb                   | 2 +-
 app/workers/activitypub/distribution_worker.rb            | 2 +-
 app/workers/feed_insert_worker.rb                         | 2 +-
 app/workers/scheduler/user_cleanup_scheduler.rb           | 2 +-
 config/environments/test.rb                               | 3 +++
 16 files changed, 25 insertions(+), 22 deletions(-)

(limited to 'app/services')

diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 106fc8224..98b1776ea 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -56,7 +56,7 @@ class Api::V1::StatusesController < Api::BaseController
     authorize @status, :destroy?
 
     @status.discard
-    RemovalWorker.perform_async(@status.id, redraft: true)
+    RemovalWorker.perform_async(@status.id, { 'redraft' => true })
     @status.account.statuses_count = @status.account.statuses_count - 1
 
     render json: @status, serializer: REST::StatusSerializer, source_requested: true
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index a861c34bc..33998c477 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -94,7 +94,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id)
 
     # Distribute into home and list feeds and notify mentioned accounts
-    ::DistributionWorker.perform_async(@status.id, silenced_account_ids: @silenced_account_ids) if @options[:override_timestamps] || @status.within_realtime_window?
+    ::DistributionWorker.perform_async(@status.id, { 'silenced_account_ids' => @silenced_account_ids }) if @options[:override_timestamps] || @status.within_realtime_window?
   end
 
   def find_existing_status
@@ -168,7 +168,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     return unless delivered_to_account.following?(@account)
 
-    FeedInsertWorker.perform_async(@status.id, delivered_to_account.id, :home)
+    FeedInsertWorker.perform_async(@status.id, delivered_to_account.id, 'home')
   end
 
   def delivered_to_account
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index c4dd9d00f..7840afee8 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -59,7 +59,7 @@ class FeedManager
     return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
 
     trim(:home, account.id)
-    PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}", update: update) if push_update_required?("timeline:#{account.id}")
+    PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}", { 'update' => update }) if push_update_required?("timeline:#{account.id}")
     true
   end
 
@@ -84,7 +84,7 @@ class FeedManager
     return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
 
     trim(:list, list.id)
-    PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", update: update) if push_update_required?("timeline:list:#{list.id}")
+    PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", { 'update' => update }) if push_update_required?("timeline:list:#{list.id}")
     true
   end
 
diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb
index 319deff98..85822214b 100644
--- a/app/models/admin/status_batch_action.rb
+++ b/app/models/admin/status_batch_action.rb
@@ -56,7 +56,7 @@ class Admin::StatusBatchAction
     end
 
     UserMailer.warning(target_account.user, @warning).deliver_later! if target_account.local?
-    RemovalWorker.push_bulk(status_ids) { |status_id| [status_id, preserve: target_account.local?, immediate: !target_account.local?] }
+    RemovalWorker.push_bulk(status_ids) { |status_id| [status_id, { 'preserve' => target_account.local?, 'immediate' => !target_account.local? }] }
   end
 
   def handle_report!
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index 4bf1775bb..dcf155840 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -103,7 +103,7 @@ class Form::AccountBatch
     authorize(account.user, :reject?)
     log_action(:reject, account.user, username: account.username)
     account.suspend!(origin: :local)
-    AccountDeletionWorker.perform_async(account.id, reserve_username: false)
+    AccountDeletionWorker.perform_async(account.id, { 'reserve_username' => false })
   end
 
   def suspend_account(account)
diff --git a/app/services/account_statuses_cleanup_service.rb b/app/services/account_statuses_cleanup_service.rb
index cbadecc63..3918b5ba4 100644
--- a/app/services/account_statuses_cleanup_service.rb
+++ b/app/services/account_statuses_cleanup_service.rb
@@ -15,7 +15,7 @@ class AccountStatusesCleanupService < BaseService
 
     account_policy.statuses_to_delete(budget, cutoff_id, account_policy.last_inspected).reorder(nil).find_each(order: :asc) do |status|
       status.discard
-      RemovalWorker.perform_async(status.id, redraft: false)
+      RemovalWorker.perform_async(status.id, { 'redraft' => false })
       num_deleted += 1
       last_deleted = status.id
     end
diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb
index e22ea1ab6..977928127 100644
--- a/app/services/activitypub/process_status_update_service.rb
+++ b/app/services/activitypub/process_status_update_service.rb
@@ -266,7 +266,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   end
 
   def broadcast_updates!
-    ::DistributionWorker.perform_async(@status.id, update: true)
+    ::DistributionWorker.perform_async(@status.id, { 'update' => true })
   end
 
   def queue_poll_notifications!
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 3a2e82b6d..4f847e293 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -59,7 +59,7 @@ class FanOutOnWriteService < BaseService
   def notify_mentioned_accounts!
     @status.active_mentions.where.not(id: @options[:silenced_account_ids] || []).joins(:account).merge(Account.local).select(:id, :account_id).reorder(nil).find_in_batches do |mentions|
       LocalNotificationWorker.push_bulk(mentions) do |mention|
-        [mention.account_id, mention.id, 'Mention', :mention]
+        [mention.account_id, mention.id, 'Mention', 'mention']
       end
     end
   end
@@ -67,7 +67,7 @@ class FanOutOnWriteService < BaseService
   def deliver_to_all_followers!
     @account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers|
       FeedInsertWorker.push_bulk(followers) do |follower|
-        [@status.id, follower.id, :home, update: update?]
+        [@status.id, follower.id, 'home', { 'update' => update? }]
       end
     end
   end
@@ -75,7 +75,7 @@ class FanOutOnWriteService < BaseService
   def deliver_to_lists!
     @account.lists_for_local_distribution.select(:id).reorder(nil).find_in_batches do |lists|
       FeedInsertWorker.push_bulk(lists) do |list|
-        [@status.id, list.id, :list, update: update?]
+        [@status.id, list.id, 'list', { 'update' => update? }]
       end
     end
   end
@@ -83,7 +83,7 @@ class FanOutOnWriteService < BaseService
   def deliver_to_mentioned_followers!
     @status.mentions.joins(:account).merge(@account.followers_for_local_distribution).select(:id, :account_id).reorder(nil).find_in_batches do |mentions|
       FeedInsertWorker.push_bulk(mentions) do |mention|
-        [@status.id, mention.account_id, :home, update: update?]
+        [@status.id, mention.account_id, 'home', { 'update' => update? }]
       end
     end
   end
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 329262cca..ed28e1371 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -68,7 +68,7 @@ class FollowService < BaseService
     follow_request = @source_account.request_follow!(@target_account, reblogs: @options[:reblogs], notify: @options[:notify], rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit])
 
     if @target_account.local?
-      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, :follow_request)
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
     elsif @target_account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url)
     end
@@ -79,7 +79,7 @@ class FollowService < BaseService
   def direct_follow!
     follow = @source_account.follow!(@target_account, reblogs: @options[:reblogs], notify: @options[:notify], rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit])
 
-    LocalNotificationWorker.perform_async(@target_account.id, follow.id, follow.class.name, :follow)
+    LocalNotificationWorker.perform_async(@target_account.id, follow.id, follow.class.name, 'follow')
     MergeWorker.perform_async(@target_account.id, @source_account.id)
 
     follow
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index 74ad5b79f..8e6640b9d 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -76,7 +76,7 @@ class ImportService < BaseService
         if presence_hash[target_account.acct]
           items.delete(target_account.acct)
           extra = presence_hash[target_account.acct][1]
-          Import::RelationshipWorker.perform_async(@account.id, target_account.acct, action, extra)
+          Import::RelationshipWorker.perform_async(@account.id, target_account.acct, action, extra.stringify_keys)
         else
           Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action)
         end
@@ -87,7 +87,7 @@ class ImportService < BaseService
     tail_items = items - head_items
 
     Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
-      [@account.id, acct, action, extra]
+      [@account.id, acct, action, extra.stringify_keys]
     end
   end
 
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index ece91847a..2d1265f10 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -47,7 +47,7 @@ class ReblogService < BaseService
     reblogged_status = reblog.reblog
 
     if reblogged_status.account.local?
-      LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name, :reblog)
+      LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name, 'reblog')
     elsif reblogged_status.account.activitypub? && !reblogged_status.account.following?(reblog.account)
       ActivityPub::DeliveryWorker.perform_async(build_json(reblog), reblog.account_id, reblogged_status.account.inbox_url)
     end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index b266c019e..3a372ef2a 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -143,7 +143,7 @@ class ResolveAccountService < BaseService
 
   def queue_deletion!
     @account.suspend!(origin: :remote)
-    AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true)
+    AccountDeletionWorker.perform_async(@account.id, { 'reserve_username' => false, 'skip_activitypub' => true })
   end
 
   def lock_options
diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb
index 17c108461..575e11025 100644
--- a/app/workers/activitypub/distribution_worker.rb
+++ b/app/workers/activitypub/distribution_worker.rb
@@ -27,6 +27,6 @@ class ActivityPub::DistributionWorker < ActivityPub::RawDistributionWorker
   end
 
   def options
-    { synchronize_followers: @status.private_visibility? }
+    { 'synchronize_followers' => @status.private_visibility? }
   end
 end
diff --git a/app/workers/feed_insert_worker.rb b/app/workers/feed_insert_worker.rb
index 0122be95d..6e3472d57 100644
--- a/app/workers/feed_insert_worker.rb
+++ b/app/workers/feed_insert_worker.rb
@@ -3,7 +3,7 @@
 class FeedInsertWorker
   include Sidekiq::Worker
 
-  def perform(status_id, id, type = :home, options = {})
+  def perform(status_id, id, type = 'home', options = {})
     @type      = type.to_sym
     @status    = Status.find(status_id)
     @options   = options.symbolize_keys
diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb
index d06b637f9..750d2127b 100644
--- a/app/workers/scheduler/user_cleanup_scheduler.rb
+++ b/app/workers/scheduler/user_cleanup_scheduler.rb
@@ -29,7 +29,7 @@ class Scheduler::UserCleanupScheduler
   def clean_discarded_statuses!
     Status.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses|
       RemovalWorker.push_bulk(statuses) do |status|
-        [status.id, { immediate: true }]
+        [status.id, { 'immediate' => true }]
       end
     end
   end
diff --git a/config/environments/test.rb b/config/environments/test.rb
index a35cadcfa..ef3cb2e48 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -70,3 +70,6 @@ if ENV['PAM_ENABLED'] == 'true'
       env: { email: 'pam@example.com' }
     }
 end
+
+# Catch serialization warnings early
+Sidekiq.strict_args!
-- 
cgit 


From 94a39f6b68e236993fe5b89a697b17c8535f6ea0 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 28 Jan 2022 09:07:56 +0100
Subject: Fix Sidekiq warning when pushing DMs to direct timeline

---
 app/services/fan_out_on_write_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 63597539e..46feec5aa 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -92,7 +92,7 @@ class FanOutOnWriteService < BaseService
 
   def deliver_to_direct_timelines!
     FeedInsertWorker.push_bulk(@status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? }) do |account|
-      [@status.id, account.id, :direct, update: update?]
+      [@status.id, account.id, 'direct', { 'update' => update? }]
     end
   end
 
-- 
cgit