From 216b85b053d091306e3311a21f5b050f70a75130 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 14 Dec 2020 09:06:34 +0100
Subject: Fix performance on instances list in admin UI (#15282)

- Reduce duplicate queries
- Remove n+1 queries
- Add accounts count to detailed view
- Add separate action log entry for updating existing domain blocks
---
 app/models/account.rb                        |  6 +--
 app/models/concerns/domain_materializable.rb | 13 ++++++
 app/models/domain_allow.rb                   |  1 +
 app/models/domain_block.rb                   |  1 +
 app/models/instance.rb                       | 63 ++++++++++++++++++++++------
 app/models/instance_filter.rb                | 31 +++++++++-----
 app/models/unavailable_domain.rb             |  2 +
 7 files changed, 88 insertions(+), 29 deletions(-)
 create mode 100644 app/models/concerns/domain_materializable.rb

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index ed11a514d..e21b353e9 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -67,6 +67,7 @@ class Account < ApplicationRecord
   include Paginable
   include AccountCounters
   include DomainNormalizable
+  include DomainMaterializable
   include AccountMerging
 
   TRUST_LEVELS = {
@@ -103,7 +104,6 @@ class Account < ApplicationRecord
   scope :bots, -> { where(actor_type: %w(Application Service)) }
   scope :groups, -> { where(actor_type: 'Group') }
   scope :alphabetic, -> { order(domain: :asc, username: :asc) }
-  scope :by_domain_accounts, -> { group(:domain).select(:domain, 'COUNT(*) AS accounts_count').order('accounts_count desc') }
   scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) }
   scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) }
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
@@ -438,10 +438,6 @@ class Account < ApplicationRecord
       super - %w(statuses_count following_count followers_count)
     end
 
-    def domains
-      reorder(nil).pluck(Arel.sql('distinct accounts.domain'))
-    end
-
     def inboxes
       urls = reorder(nil).where(protocol: :activitypub).group(:preferred_inbox_url).pluck(Arel.sql("coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url) AS preferred_inbox_url"))
       DeliveryFailureTracker.without_unavailable(urls)
diff --git a/app/models/concerns/domain_materializable.rb b/app/models/concerns/domain_materializable.rb
new file mode 100644
index 000000000..88337f8c0
--- /dev/null
+++ b/app/models/concerns/domain_materializable.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module DomainMaterializable
+  extend ActiveSupport::Concern
+
+  included do
+    after_create_commit :refresh_instances_view
+  end
+
+  def refresh_instances_view
+    Instance.refresh unless domain.nil? || Instance.where(domain: domain).exists?
+  end
+end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
index 5fe0e3a29..4b0a89c18 100644
--- a/app/models/domain_allow.rb
+++ b/app/models/domain_allow.rb
@@ -12,6 +12,7 @@
 
 class DomainAllow < ApplicationRecord
   include DomainNormalizable
+  include DomainMaterializable
 
   validates :domain, presence: true, uniqueness: true, domain: true
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 2b18e01fa..829d7583b 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -16,6 +16,7 @@
 
 class DomainBlock < ApplicationRecord
   include DomainNormalizable
+  include DomainMaterializable
 
   enum severity: [:silence, :suspend, :noop]
 
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 3c740f8a2..29be03662 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -1,26 +1,63 @@
 # frozen_string_literal: true
+# == Schema Information
+#
+# Table name: instances
+#
+#  domain         :string           primary key
+#  accounts_count :bigint(8)
+#
 
-class Instance
-  include ActiveModel::Model
+class Instance < ApplicationRecord
+  self.primary_key = :domain
 
-  attr_accessor :domain, :accounts_count, :domain_block
+  has_many :accounts, foreign_key: :domain, primary_key: :domain
 
-  def initialize(resource)
-    @domain         = resource.domain
-    @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil
-    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
-    @domain_allow   = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain)
+  belongs_to :domain_block, foreign_key: :domain, primary_key: :domain
+  belongs_to :domain_allow, foreign_key: :domain, primary_key: :domain
+
+  scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+
+  def self.refresh
+    Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false)
   end
 
-  def countable?
-    @accounts_count.present?
+  def readonly?
+    true
   end
 
-  def to_param
-    domain
+  def delivery_failure_tracker
+    @delivery_failure_tracker ||= DeliveryFailureTracker.new(domain)
+  end
+
+  def following_count
+    @following_count ||= Follow.where(account: accounts).count
+  end
+
+  def followers_count
+    @followers_count ||= Follow.where(target_account: accounts).count
+  end
+
+  def reports_count
+    @reports_count ||= Report.where(target_account: accounts).count
   end
 
-  def cache_key
+  def blocks_count
+    @blocks_count ||= Block.where(target_account: accounts).count
+  end
+
+  def public_comment
+    domain_block&.public_comment
+  end
+
+  def private_comment
+    domain_block&.private_comment
+  end
+
+  def media_storage
+    @media_storage ||= MediaAttachment.where(account: accounts).sum(:file_file_size)
+  end
+
+  def to_param
     domain
   end
 end
diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb
index 9c467bc27..0598d8fea 100644
--- a/app/models/instance_filter.rb
+++ b/app/models/instance_filter.rb
@@ -13,18 +13,27 @@ class InstanceFilter
   end
 
   def results
-    if params[:limited].present?
-      scope = DomainBlock
-      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
-      scope.order(id: :desc)
-    elsif params[:allowed].present?
-      scope = DomainAllow
-      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
-      scope.order(id: :desc)
+    scope = Instance.includes(:domain_block, :domain_allow).order(accounts_count: :desc)
+
+    params.each do |key, value|
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'limited'
+      Instance.joins(:domain_block).reorder(Arel.sql('domain_blocks.id desc'))
+    when 'allowed'
+      Instance.joins(:domain_allow).reorder(Arel.sql('domain_allows.id desc'))
+    when 'by_domain'
+      Instance.matches_domain(value)
     else
-      scope = Account.remote
-      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
-      scope.by_domain_accounts
+      raise "Unknown filter: #{key}"
     end
   end
 end
diff --git a/app/models/unavailable_domain.rb b/app/models/unavailable_domain.rb
index e2918b586..5e8870bde 100644
--- a/app/models/unavailable_domain.rb
+++ b/app/models/unavailable_domain.rb
@@ -12,6 +12,8 @@
 class UnavailableDomain < ApplicationRecord
   include DomainNormalizable
 
+  validates :domain, presence: true, uniqueness: true
+
   after_commit :reset_cache!
 
   private
-- 
cgit 


From 47e507fa61be6dc39dd9821e1d07c33e993cc246 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 14 Dec 2020 10:03:09 +0100
Subject: Add ability to require invite request text (#15326)

Fixes #15273

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/javascript/packs/admin.js              | 26 ++++++++++++++++++++++++++
 app/javascript/styles/mastodon/forms.scss  | 15 ++++++++++-----
 app/models/form/admin_settings.rb          |  2 ++
 app/models/user.rb                         |  3 ++-
 app/views/about/_registration.html.haml    |  2 +-
 app/views/admin/settings/edit.html.haml    |  6 ++++++
 app/views/auth/registrations/new.html.haml |  2 +-
 config/locales/en.yml                      |  3 +++
 config/settings.yml                        |  1 +
 9 files changed, 52 insertions(+), 8 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/packs/admin.js b/app/javascript/packs/admin.js
index 1c44fb45b..8fd1b8a8e 100644
--- a/app/javascript/packs/admin.js
+++ b/app/javascript/packs/admin.js
@@ -67,10 +67,36 @@ const onEnableBootstrapTimelineAccountsChange = (target) => {
 
 delegate(document, '#form_admin_settings_enable_bootstrap_timeline_accounts', 'change', ({ target }) => onEnableBootstrapTimelineAccountsChange(target));
 
+const onChangeRegistrationMode = (target) => {
+  const enabled = target.value === 'approved';
+
+  [].forEach.call(document.querySelectorAll('#form_admin_settings_require_invite_text'), (input) => {
+    input.disabled = !enabled;
+    if (enabled) {
+      let element = input;
+      do {
+        element.classList.remove('disabled');
+        element = element.parentElement;
+      } while (element && !element.classList.contains('fields-group'));
+    } else {
+      let element = input;
+      do {
+        element.classList.add('disabled');
+        element = element.parentElement;
+      } while (element && !element.classList.contains('fields-group'));
+    }
+  });
+};
+
+delegate(document, '#form_admin_settings_registrations_mode', 'change', ({ target }) => onChangeRegistrationMode(target));
+
 ready(() => {
   const domainBlockSeverityInput = document.getElementById('domain_block_severity');
   if (domainBlockSeverityInput) onDomainBlockSeverityChange(domainBlockSeverityInput);
 
   const enableBootstrapTimelineAccounts = document.getElementById('form_admin_settings_enable_bootstrap_timeline_accounts');
   if (enableBootstrapTimelineAccounts) onEnableBootstrapTimelineAccountsChange(enableBootstrapTimelineAccounts);
+
+  const registrationMode = document.getElementById('form_admin_settings_registrations_mode');
+  if (registrationMode) onChangeRegistrationMode(registrationMode);
 });
diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss
index 92d89e6f2..e0604303b 100644
--- a/app/javascript/styles/mastodon/forms.scss
+++ b/app/javascript/styles/mastodon/forms.scss
@@ -377,11 +377,6 @@ code {
       box-shadow: none;
     }
 
-    &:focus:invalid:not(:placeholder-shown),
-    &:required:invalid:not(:placeholder-shown) {
-      border-color: lighten($error-red, 12%);
-    }
-
     &:required:valid {
       border-color: $valid-value-color;
     }
@@ -397,6 +392,16 @@ code {
     }
   }
 
+  input[type=text],
+  input[type=number],
+  input[type=email],
+  input[type=password] {
+    &:focus:invalid:not(:placeholder-shown),
+    &:required:invalid:not(:placeholder-shown) {
+      border-color: lighten($error-red, 12%);
+    }
+  }
+
   .input.field_with_errors {
     label {
       color: lighten($error-red, 12%);
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 390836f28..e9f78da21 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -35,6 +35,7 @@ class Form::AdminSettings
     show_domain_blocks
     show_domain_blocks_rationale
     noindex
+    require_invite_text
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -51,6 +52,7 @@ class Form::AdminSettings
     trends
     trendable_by_default
     noindex
+    require_invite_text
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/models/user.rb b/app/models/user.rb
index 981dc6d47..6088f1994 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -82,7 +82,8 @@ class User < ApplicationRecord
   has_many :webauthn_credentials, dependent: :destroy
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
-  accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
+  accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text }
+  validates :invite_request, presence: true, on: :create, if: -> { Setting.require_invite_text }
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
   validates_with BlacklistedEmailValidator, on: :create
diff --git a/app/views/about/_registration.html.haml b/app/views/about/_registration.html.haml
index 6160ca4d4..e4d614d71 100644
--- a/app/views/about/_registration.html.haml
+++ b/app/views/about/_registration.html.haml
@@ -16,7 +16,7 @@
     - if approved_registrations?
       .fields-group
         = f.simple_fields_for :invite_request do |invite_request_fields|
-          = invite_request_fields.input :text, as: :text, wrapper: :with_block_label, required: false
+          = invite_request_fields.input :text, as: :text, wrapper: :with_block_label, required: Setting.require_invite_text
 
     .fields-group
       = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path), required: true, disabled: closed_registrations?
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 9e28766b1..a162490b5 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -43,6 +43,12 @@
 
   %hr.spacer/
 
+  .fields-group
+    = f.input :require_invite_text, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.require_invite_text.title'), hint: t('admin.settings.registrations.require_invite_text.desc_html'), disabled: !approved_registrations?
+  .fields-group
+
+  %hr.spacer/
+
   .fields-group
     = f.input :enable_bootstrap_timeline_accounts, as: :boolean, wrapper: :with_label, label: t('admin.settings.enable_bootstrap_timeline_accounts.title')
   .fields-group
diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml
index de541847f..6981195ed 100644
--- a/app/views/auth/registrations/new.html.haml
+++ b/app/views/auth/registrations/new.html.haml
@@ -31,7 +31,7 @@
   - if approved_registrations? && !@invite.present?
     .fields-group
       = f.simple_fields_for :invite_request, resource.invite_request || resource.build_invite_request do |invite_request_fields|
-        = invite_request_fields.input :text, as: :text, wrapper: :with_block_label, required: false
+        = invite_request_fields.input :text, as: :text, wrapper: :with_block_label, required: Setting.require_invite_text
 
   = f.input :invite_code, as: :hidden
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index f89f50e4d..114f86fd1 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -586,6 +586,9 @@ en:
         min_invite_role:
           disabled: No one
           title: Allow invitations by
+        require_invite_text:
+          desc_html: When registrations require manual approval, make the “Why do you want to join?” invite request text mandatory rather than optional
+          title: Require new users to fill an invite request text
       registrations_mode:
         modes:
           approved: Approval required for sign up
diff --git a/config/settings.yml b/config/settings.yml
index 217745f28..9cf68a096 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -70,6 +70,7 @@ defaults: &defaults
   spam_check_enabled: true
   show_domain_blocks: 'disabled'
   show_domain_blocks_rationale: 'disabled'
+  require_invite_text: false
 
 development:
   <<: *defaults
-- 
cgit 


From 1390cc194bd07b85fe36e31a0cab22981315974d Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 15 Dec 2020 04:30:15 +0100
Subject: Add indication to admin UI of whether a report has been forwarded
 (#13237)

* Add indication to admin UI of whether a report has been forwarded

* Rework how forwarded status is displayed

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/models/report.rb                                  |  1 +
 app/services/report_service.rb                        |  3 ++-
 app/views/admin/reports/index.html.haml               |  4 ++++
 app/views/admin/reports/show.html.haml                | 10 ++++++++++
 config/locales/en.yml                                 |  2 ++
 db/migrate/20200309150742_add_forwarded_to_reports.rb |  5 +++++
 db/schema.rb                                          |  1 +
 7 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20200309150742_add_forwarded_to_reports.rb

(limited to 'app/models')

diff --git a/app/models/report.rb b/app/models/report.rb
index f31bcfd2e..cd08120e4 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -14,6 +14,7 @@
 #  target_account_id          :bigint(8)        not null
 #  assigned_account_id        :bigint(8)
 #  uri                        :string
+#  forwarded                  :boolean
 #
 
 class Report < ApplicationRecord
diff --git a/app/services/report_service.rb b/app/services/report_service.rb
index 1e955c1e7..9d9c7d6c9 100644
--- a/app/services/report_service.rb
+++ b/app/services/report_service.rb
@@ -24,7 +24,8 @@ class ReportService < BaseService
       target_account: @target_account,
       status_ids: @status_ids,
       comment: @comment,
-      uri: @options[:uri]
+      uri: @options[:uri],
+      forwarded: ActiveModel::Type::Boolean.new.cast(@options[:forward])
     )
   end
 
diff --git a/app/views/admin/reports/index.html.haml b/app/views/admin/reports/index.html.haml
index bb441380e..721c55f71 100644
--- a/app/views/admin/reports/index.html.haml
+++ b/app/views/admin/reports/index.html.haml
@@ -59,6 +59,10 @@
                 = fa_icon('camera')
                 = report.media_attachments.count
 
+              - if report.forwarded?
+                ·
+                = t('admin.reports.forwarded_to', domain: target_account.domain)
+
           .report-card__summary__item__assigned
             - if report.assigned_account.present?
               = admin_account_link_to report.assigned_account
diff --git a/app/views/admin/reports/show.html.haml b/app/views/admin/reports/show.html.haml
index 2681419ca..b060c553f 100644
--- a/app/views/admin/reports/show.html.haml
+++ b/app/views/admin/reports/show.html.haml
@@ -46,6 +46,16 @@
         %td{ colspan: 2 }
           - if @report.action_taken?
             = table_link_to 'envelope-open', t('admin.reports.reopen'), admin_report_path(@report, outcome: 'reopen'), method: :put
+      - unless @report.target_account.local?
+        %tr
+          %th= t('admin.reports.forwarded')
+          %td{ colspan: 3 }
+            - if @report.forwarded.nil?
+              \-
+            - elsif @report.forwarded?
+              = t('simple_form.yes')
+            - else
+              = t('simple_form.no')
       - if !@report.action_taken_by_account.nil?
         %tr
           %th= t('admin.reports.action_taken_by')
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 114f86fd1..3ff01ac86 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -517,6 +517,8 @@ en:
       comment:
         none: None
       created_at: Reported
+      forwarded: Forwarded
+      forwarded_to: Forwarded to %{domain}
       mark_as_resolved: Mark as resolved
       mark_as_unresolved: Mark as unresolved
       notes:
diff --git a/db/migrate/20200309150742_add_forwarded_to_reports.rb b/db/migrate/20200309150742_add_forwarded_to_reports.rb
new file mode 100644
index 000000000..df278240b
--- /dev/null
+++ b/db/migrate/20200309150742_add_forwarded_to_reports.rb
@@ -0,0 +1,5 @@
+class AddForwardedToReports < ActiveRecord::Migration[5.2]
+  def change
+    add_column :reports, :forwarded, :boolean
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2f9d369be..55822a4b3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -717,6 +717,7 @@ ActiveRecord::Schema.define(version: 2020_12_06_004238) do
     t.bigint "target_account_id", null: false
     t.bigint "assigned_account_id"
     t.string "uri"
+    t.boolean "forwarded"
     t.index ["account_id"], name: "index_reports_on_account_id"
     t.index ["target_account_id"], name: "index_reports_on_target_account_id"
   end
-- 
cgit 


From 83579695593c941ab39ab545c10a7f711bf715fc Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 15 Dec 2020 17:23:58 +0100
Subject: Fix admins being able to suspend their instance actor (#14567)

* Fix admin being able to suspend their own instance account

* Add text about the instance's own actor in admin view

* Change instance actor notice from flash message to template

* Do not list local instance actor in account moderation list
---
 app/models/account.rb                   | 3 ++-
 app/models/account_filter.rb            | 2 +-
 app/policies/account_policy.rb          | 4 ++--
 app/views/admin/accounts/show.html.haml | 4 ++++
 config/locales/en.yml                   | 1 +
 spec/models/account_filter_spec.rb      | 2 +-
 6 files changed, 11 insertions(+), 5 deletions(-)

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index e21b353e9..80eb92a71 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -100,6 +100,7 @@ class Account < ApplicationRecord
   scope :sensitized, -> { where.not(sensitized_at: nil) }
   scope :without_suspended, -> { where(suspended_at: nil) }
   scope :without_silenced, -> { where(silenced_at: nil) }
+  scope :without_instance_actor, -> { where.not(id: -99) }
   scope :recent, -> { reorder(id: :desc) }
   scope :bots, -> { where(actor_type: %w(Application Service)) }
   scope :groups, -> { where(actor_type: 'Group') }
@@ -222,7 +223,7 @@ class Account < ApplicationRecord
   end
 
   def suspended?
-    suspended_at.present?
+    suspended_at.present? && !instance_actor?
   end
 
   def suspended_permanently?
diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb
index 7b6012e0f..2b001385f 100644
--- a/app/models/account_filter.rb
+++ b/app/models/account_filter.rb
@@ -45,7 +45,7 @@ class AccountFilter
   def scope_for(key, value)
     case key.to_s
     when 'local'
-      Account.local
+      Account.local.without_instance_actor
     when 'remote'
       Account.remote
     when 'by_domain'
diff --git a/app/policies/account_policy.rb b/app/policies/account_policy.rb
index 262ada42e..672e1786b 100644
--- a/app/policies/account_policy.rb
+++ b/app/policies/account_policy.rb
@@ -14,7 +14,7 @@ class AccountPolicy < ApplicationPolicy
   end
 
   def suspend?
-    staff? && !record.user&.staff?
+    staff? && !record.user&.staff? && !record.instance_actor?
   end
 
   def destroy?
@@ -62,6 +62,6 @@ class AccountPolicy < ApplicationPolicy
   end
 
   def memorialize?
-    admin? && !record.user&.admin?
+    admin? && !record.user&.admin? && !record.instance_actor?
   end
 end
diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml
index ae527cc23..27e1f80a7 100644
--- a/app/views/admin/accounts/show.html.haml
+++ b/app/views/admin/accounts/show.html.haml
@@ -1,6 +1,10 @@
 - content_for :page_title do
   = @account.acct
 
+- if @account.instance_actor?
+  .flash-message.notice
+    %strong= t('accounts.instance_actor_flash')
+
 = render 'application/card', account: @account
 
 - account = @account
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 1f5798fcc..83f75b28f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -60,6 +60,7 @@ en:
       one: Follower
       other: Followers
     following: Following
+    instance_actor_flash: This account is a virtual actor used to represent the server itself and not any individual user. It is used for federation purposes and should not be suspended.
     joined: Joined %{date}
     last_active: last active
     link_verified_on: Ownership of this link was checked on %{date}
diff --git a/spec/models/account_filter_spec.rb b/spec/models/account_filter_spec.rb
index 176a0eeac..0cdb373f6 100644
--- a/spec/models/account_filter_spec.rb
+++ b/spec/models/account_filter_spec.rb
@@ -5,7 +5,7 @@ describe AccountFilter do
     it 'defaults to recent local not-suspended account list' do
       filter = described_class.new({})
 
-      expect(filter.results).to eq Account.local.recent.without_suspended
+      expect(filter.results).to eq Account.local.without_instance_actor.recent.without_suspended
     end
   end
 
-- 
cgit 


From 8a95867693dfe072241d5ddcd0ba7ed8546eab1e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 18 Dec 2020 08:30:41 +0100
Subject: Add option to obfuscate domain name in public list of domain blocks
 (#15355)

- Replace the middle of the domain with * characters (except for periods)
- Add SHA-256 digest of the domain name in tooltip
---
 app/controllers/admin/domain_blocks_controller.rb    |  4 ++--
 app/models/domain_block.rb                           | 20 ++++++++++++++++++++
 app/views/about/_domain_blocks.html.haml             |  2 +-
 app/views/admin/domain_blocks/edit.html.haml         |  3 +++
 app/views/admin/domain_blocks/new.html.haml          |  3 +++
 config/locales/en.yml                                |  2 ++
 .../20201218054746_add_obfuscate_to_domain_blocks.rb | 15 +++++++++++++++
 db/schema.rb                                         |  3 ++-
 8 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 db/migrate/20201218054746_add_obfuscate_to_domain_blocks.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 6a5b41a74..ba927b04a 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -74,11 +74,11 @@ module Admin
     end
 
     def update_params
-      params.require(:domain_block).permit(:severity, :reject_media, :reject_reports, :private_comment, :public_comment)
+      params.require(:domain_block).permit(:severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate)
     end
 
     def resource_params
-      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment)
+      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate)
     end
   end
 end
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 829d7583b..bba04c603 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -12,6 +12,7 @@
 #  reject_reports  :boolean          default(FALSE), not null
 #  private_comment :text
 #  public_comment  :text
+#  obfuscate       :boolean          default(FALSE), not null
 #
 
 class DomainBlock < ApplicationRecord
@@ -73,4 +74,23 @@ class DomainBlock < ApplicationRecord
     scope = suspend? ? accounts.where(suspended_at: created_at) : accounts.where(silenced_at: created_at)
     scope.count
   end
+
+  def public_domain
+    return domain unless obfuscate?
+
+    length        = domain.size
+    visible_ratio = length / 4
+
+    domain.chars.map.with_index do |chr, i|
+      if i > visible_ratio && i < length - visible_ratio && chr != '.'
+        '*'
+      else
+        chr
+      end
+    end.join
+  end
+
+  def domain_digest
+    Digest::SHA256.hexdigest(domain)
+  end
 end
diff --git a/app/views/about/_domain_blocks.html.haml b/app/views/about/_domain_blocks.html.haml
index e0c5df41d..35a30f16e 100644
--- a/app/views/about/_domain_blocks.html.haml
+++ b/app/views/about/_domain_blocks.html.haml
@@ -7,6 +7,6 @@
     - domain_blocks.each do |domain_block|
       %tr
         %td.nowrap
-          %span{ title: domain_block.domain }= domain_block.domain
+          %span{ title: "SHA-256: #{domain_block.domain_digest}" }= domain_block.public_domain
         %td
           = domain_block.public_comment if display_blocks_rationale?
diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml
index d5868070a..6fe2edc82 100644
--- a/app/views/admin/domain_blocks/edit.html.haml
+++ b/app/views/admin/domain_blocks/edit.html.haml
@@ -20,6 +20,9 @@
   .fields-group
     = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
 
+  .fields-group
+    = f.input :obfuscate, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.obfuscate'), hint: I18n.t('admin.domain_blocks.obfuscate_hint')
+
   .field-group
     = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
 
diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml
index f503f9b77..8b78f71f2 100644
--- a/app/views/admin/domain_blocks/new.html.haml
+++ b/app/views/admin/domain_blocks/new.html.haml
@@ -20,6 +20,9 @@
   .fields-group
     = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
 
+  .fields-group
+    = f.input :obfuscate, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.obfuscate'), hint: I18n.t('admin.domain_blocks.obfuscate_hint')
+
   .field-group
     = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 83f75b28f..e91f4344f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -402,6 +402,8 @@ en:
           silence: Silence
           suspend: Suspend
         title: New domain block
+      obfuscate: Obfuscate domain name
+      obfuscate_hint: Partially obfuscate the domain name in the list if advertising the list of domain limitations is enabled
       private_comment: Private comment
       private_comment_hint: Comment about this domain limitation for internal use by the moderators.
       public_comment: Public comment
diff --git a/db/migrate/20201218054746_add_obfuscate_to_domain_blocks.rb b/db/migrate/20201218054746_add_obfuscate_to_domain_blocks.rb
new file mode 100644
index 000000000..26f4ddb85
--- /dev/null
+++ b/db/migrate/20201218054746_add_obfuscate_to_domain_blocks.rb
@@ -0,0 +1,15 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddObfuscateToDomainBlocks < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :domain_blocks, :obfuscate, :boolean, default: false, allow_null: false }
+  end
+
+  def down
+    remove_column :domain_blocks, :obfuscate
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 55822a4b3..18bf1d4ca 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_12_06_004238) do
+ActiveRecord::Schema.define(version: 2020_12_18_054746) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -360,6 +360,7 @@ ActiveRecord::Schema.define(version: 2020_12_06_004238) do
     t.boolean "reject_reports", default: false, null: false
     t.text "private_comment"
     t.text "public_comment"
+    t.boolean "obfuscate", default: false, null: false
     t.index ["domain"], name: "index_domain_blocks_on_domain", unique: true
   end
 
-- 
cgit 


From eb35be0431b2cdd2bbf3339beb9c5a0839e1088b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 18 Dec 2020 09:18:31 +0100
Subject: Fix follow limit preventing re-following of a moved account (#14207)

---
 app/models/follow.rb               |  2 +-
 app/models/follow_request.rb       |  2 +-
 app/models/import.rb               |  1 +
 app/services/import_service.rb     |  7 ++----
 app/validators/import_validator.rb | 44 ++++++++++++++++++++++++++++++++++++++
 config/locales/en.yml              |  2 ++
 spec/models/follow_spec.rb         |  2 +-
 spec/models/import_spec.rb         | 10 +++++++++
 8 files changed, 62 insertions(+), 8 deletions(-)
 create mode 100644 app/validators/import_validator.rb

(limited to 'app/models')

diff --git a/app/models/follow.rb b/app/models/follow.rb
index 55a9da792..69a1722b3 100644
--- a/app/models/follow.rb
+++ b/app/models/follow.rb
@@ -26,7 +26,7 @@ class Follow < ApplicationRecord
   has_one :notification, as: :activity, dependent: :destroy
 
   validates :account_id, uniqueness: { scope: :target_account_id }
-  validates_with FollowLimitValidator, on: :create
+  validates_with FollowLimitValidator, on: :create, if: :rate_limit?
 
   scope :recent, -> { reorder(id: :desc) }
 
diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb
index c1f19149b..2d2a77b59 100644
--- a/app/models/follow_request.rb
+++ b/app/models/follow_request.rb
@@ -26,7 +26,7 @@ class FollowRequest < ApplicationRecord
   has_one :notification, as: :activity, dependent: :destroy
 
   validates :account_id, uniqueness: { scope: :target_account_id }
-  validates_with FollowLimitValidator, on: :create
+  validates_with FollowLimitValidator, on: :create, if: :rate_limit?
 
   def authorize!
     account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri)
diff --git a/app/models/import.rb b/app/models/import.rb
index 702453289..00a54892e 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -27,6 +27,7 @@ class Import < ApplicationRecord
   enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks]
 
   validates :type, presence: true
+  validates_with ImportValidator, on: :create
 
   has_attached_file :data
   validates_attachment_content_type :data, content_type: FILE_TYPES
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index 288e47f1e..0c6ef2238 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -27,7 +27,7 @@ class ImportService < BaseService
 
   def import_follows!
     parse_import_data!(['Account address'])
-    import_relationships!('follow', 'unfollow', @account.following, follow_limit, reblogs: { header: 'Show boosts', default: true })
+    import_relationships!('follow', 'unfollow', @account.following, ROWS_PROCESSING_LIMIT, reblogs: { header: 'Show boosts', default: true })
   end
 
   def import_blocks!
@@ -85,6 +85,7 @@ class ImportService < BaseService
 
     head_items = items.uniq { |acct, _| acct.split('@')[1] }
     tail_items = items - head_items
+
     Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
       [@account.id, acct, action, extra]
     end
@@ -133,10 +134,6 @@ class ImportService < BaseService
     Paperclip.io_adapters.for(@import.data).read
   end
 
-  def follow_limit
-    FollowLimitValidator.limit_for_account(@account)
-  end
-
   def relations_map_for_account(account, account_ids)
     {
       blocking: {},
diff --git a/app/validators/import_validator.rb b/app/validators/import_validator.rb
new file mode 100644
index 000000000..a182abfa5
--- /dev/null
+++ b/app/validators/import_validator.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class ImportValidator < ActiveModel::Validator
+  KNOWN_HEADERS = [
+    'Account address',
+    '#domain',
+    '#uri',
+  ].freeze
+
+  def validate(import)
+    return if import.type.blank? || import.data.blank?
+
+    # We parse because newlines could be part of individual rows. This
+    # runs on create so we should be reading the local file here before
+    # it is uploaded to object storage or moved anywhere...
+    csv_data = CSV.parse(import.data.queued_for_write[:original].read)
+
+    row_count  = csv_data.size
+    row_count -= 1 if KNOWN_HEADERS.include?(csv_data.first&.first)
+
+    import.errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ImportService::ROWS_PROCESSING_LIMIT)) if row_count > ImportService::ROWS_PROCESSING_LIMIT
+
+    case import.type
+    when 'following'
+      validate_following_import(import, row_count)
+    end
+  end
+
+  private
+
+  def validate_following_import(import, row_count)
+    base_limit = FollowLimitValidator.limit_for_account(import.account)
+
+    limit = begin
+      if import.overwrite?
+        base_limit
+      else
+        base_limit - import.account.following_count
+      end
+    end
+
+    import.errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if row_count > limit
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index e91f4344f..18a207f5e 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -923,6 +923,8 @@ en:
     status: Verification status
     view_proof: View proof
   imports:
+    errors:
+      over_rows_processing_limit: contains more than %{count} rows
     modes:
       merge: Merge
       merge_long: Keep existing records and add new ones
diff --git a/spec/models/follow_spec.rb b/spec/models/follow_spec.rb
index 0c84e5e7b..e723a1ef2 100644
--- a/spec/models/follow_spec.rb
+++ b/spec/models/follow_spec.rb
@@ -5,7 +5,7 @@ RSpec.describe Follow, type: :model do
   let(:bob)   { Fabricate(:account, username: 'bob') }
 
   describe 'validations' do
-    subject { Follow.new(account: alice, target_account: bob) }
+    subject { Follow.new(account: alice, target_account: bob, rate_limit: true) }
 
     it 'has a valid fabricator' do
       follow = Fabricate.build(:follow)
diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb
index 321761166..a5eec1722 100644
--- a/spec/models/import_spec.rb
+++ b/spec/models/import_spec.rb
@@ -20,5 +20,15 @@ RSpec.describe Import, type: :model do
       import = Import.create(account: account, type: type)
       expect(import).to model_have_error_on_field(:data)
     end
+
+    it 'is invalid with too many rows in data' do
+      import = Import.create(account: account, type: type, data: StringIO.new("foo@bar.com\n" * (ImportService::ROWS_PROCESSING_LIMIT + 10)))
+      expect(import).to model_have_error_on_field(:data)
+    end
+
+    it 'is invalid when there are more rows when following limit' do
+      import = Import.create(account: account, type: type, data: StringIO.new("foo@bar.com\n" * (FollowLimitValidator.limit_for_account(account) + 10)))
+      expect(import).to model_have_error_on_field(:data)
+    end
   end
 end
-- 
cgit 


From 7bf3c6e57b52cd9390f2140a1cc17292c281aacf Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 20 Dec 2020 18:25:00 +0100
Subject: Fix AccountDeletionWorker crashing and clogging sidekiq queues
 (#15380)

* Fix account deletion workers being queued multiple times for a single account

* Fix poll votes being unnecessarily instantiated on poll deletion

* Fix favourites being unnecessarily instantiated on status deletion

* Remove inaccurate comments

* Delete polls instead of destroying them

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/models/poll.rb                            | 2 +-
 app/models/status.rb                          | 2 +-
 app/services/batched_remove_status_service.rb | 3 ---
 app/services/delete_account_service.rb        | 4 +++-
 app/workers/account_deletion_worker.rb        | 2 +-
 5 files changed, 6 insertions(+), 7 deletions(-)

(limited to 'app/models')

diff --git a/app/models/poll.rb b/app/models/poll.rb
index b5deafcc2..e1ca55252 100644
--- a/app/models/poll.rb
+++ b/app/models/poll.rb
@@ -25,7 +25,7 @@ class Poll < ApplicationRecord
   belongs_to :account
   belongs_to :status
 
-  has_many :votes, class_name: 'PollVote', inverse_of: :poll, dependent: :destroy
+  has_many :votes, class_name: 'PollVote', inverse_of: :poll, dependent: :delete_all
 
   has_many :notifications, as: :activity, dependent: :destroy
 
diff --git a/app/models/status.rb b/app/models/status.rb
index 96d90e1c2..f1b3b75ce 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -56,7 +56,7 @@ class Status < ApplicationRecord
   belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies, optional: true
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
-  has_many :favourites, inverse_of: :status, dependent: :destroy
+  has_many :favourites, inverse_of: :status, dependent: :delete_all
   has_many :bookmarks, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 2295a01dc..28e5468b3 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -4,8 +4,6 @@ class BatchedRemoveStatusService < BaseService
   include Redisable
 
   # Delete given statuses and reblogs of them
-  # Dispatch PuSH updates of the deleted statuses, but only local ones
-  # Dispatch Salmon deletes, unique per domain, of the deleted statuses, but only local ones
   # Remove statuses from home feeds
   # Push delete events to streaming API for home feeds and public feeds
   # @param [Enumerable<Status>] statuses A preferably batched array of statuses
@@ -19,7 +17,6 @@ class BatchedRemoveStatusService < BaseService
 
     @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
 
-    # Ensure that rendered XML reflects destroyed state
     statuses.each do |status|
       status.mark_for_mass_destruction!
       status.destroy
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index 9cb80c95a..fe9b30b17 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -122,7 +122,9 @@ class DeleteAccountService < BaseService
     @account.polls.reorder(nil).find_each do |poll|
       next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id)
 
-      poll.destroy
+      # We can safely delete the poll rather than destroy it, as any non-reported
+      # status should have been deleted already
+      poll.delete
     end
 
     associations_for_destruction.each do |association_name|
diff --git a/app/workers/account_deletion_worker.rb b/app/workers/account_deletion_worker.rb
index 98b67419d..fdf013e01 100644
--- a/app/workers/account_deletion_worker.rb
+++ b/app/workers/account_deletion_worker.rb
@@ -3,7 +3,7 @@
 class AccountDeletionWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull'
+  sidekiq_options queue: 'pull', lock: :until_executed
 
   def perform(account_id, options = {})
     reserve_username = options.with_indifferent_access.fetch(:reserve_username, true)
-- 
cgit 


From 6f51fd743590c8fd8dd95e48dc24e4472d46480b Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 21 Dec 2020 00:47:02 +0100
Subject: Fix invitation links not working when invite request text is required
 (#15385)

Fixes #15383

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/models/user.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/user.rb b/app/models/user.rb
index 6088f1994..dd96bbf8c 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -83,7 +83,7 @@ class User < ApplicationRecord
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text }
-  validates :invite_request, presence: true, on: :create, if: -> { Setting.require_invite_text }
+  validates :invite_request, presence: true, on: :create, if: -> { Setting.require_invite_text && !invited? }
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
   validates_with BlacklistedEmailValidator, on: :create
-- 
cgit 


From 43961035a906cd8bccdf4c1ac023980b37823bb3 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 21 Dec 2020 18:22:17 +0100
Subject: Fix some notifications not being deleted on poll/status deletion
 (#15402)

* Fix deleting polls not deleting notifications

* Fix fav notification deletion when deleting a toot

* Refactor DeleteAccountService spec

* Add DeleteAccountService tests for other associations and notifications

* Add favourite handling spec in status removal

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/models/status.rb                         |   2 +-
 app/services/delete_account_service.rb       |   4 +-
 spec/services/delete_account_service_spec.rb | 108 +++++++++++++++------------
 spec/services/remove_status_service_spec.rb  |  14 +++-
 4 files changed, 76 insertions(+), 52 deletions(-)

(limited to 'app/models')

diff --git a/app/models/status.rb b/app/models/status.rb
index f1b3b75ce..96d90e1c2 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -56,7 +56,7 @@ class Status < ApplicationRecord
   belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies, optional: true
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
-  has_many :favourites, inverse_of: :status, dependent: :delete_all
+  has_many :favourites, inverse_of: :status, dependent: :destroy
   has_many :bookmarks, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index fe9b30b17..fa834e775 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -123,7 +123,9 @@ class DeleteAccountService < BaseService
       next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id)
 
       # We can safely delete the poll rather than destroy it, as any non-reported
-      # status should have been deleted already
+      # status should have been deleted already, as long as we take care of
+      # notifications.
+      Notification.where(poll: poll).delete_all
       poll.delete
     end
 
diff --git a/spec/services/delete_account_service_spec.rb b/spec/services/delete_account_service_spec.rb
index d208b25b8..cd7d32d59 100644
--- a/spec/services/delete_account_service_spec.rb
+++ b/spec/services/delete_account_service_spec.rb
@@ -1,28 +1,31 @@
 require 'rails_helper'
 
 RSpec.describe DeleteAccountService, type: :service do
-  describe '#call on local account' do
-    before do
-      stub_request(:post, "https://alice.com/inbox").to_return(status: 201)
-      stub_request(:post, "https://bob.com/inbox").to_return(status: 201)
-    end
+  shared_examples 'common behavior' do
+    let!(:status) { Fabricate(:status, account: account) }
+    let!(:mention) { Fabricate(:mention, account: local_follower) }
+    let!(:status_with_mention) { Fabricate(:status, account: account, mentions: [mention]) }
+    let!(:media_attachment) { Fabricate(:media_attachment, account: account) }
+    let!(:notification) { Fabricate(:notification, account: account) }
+    let!(:favourite) { Fabricate(:favourite, account: account, status: Fabricate(:status, account: local_follower)) }
+    let!(:poll) { Fabricate(:poll, account: account) }
+    let!(:poll_vote) { Fabricate(:poll_vote, account: local_follower, poll: poll) }
+
+    let!(:active_relationship) { Fabricate(:follow, account: account, target_account: local_follower) }
+    let!(:passive_relationship) { Fabricate(:follow, account: local_follower, target_account: account) }
+    let!(:endorsement) { Fabricate(:account_pin, account: local_follower, target_account: account) }
+
+    let!(:mention_notification) { Fabricate(:notification, account: local_follower, activity: mention, type: :mention) }
+    let!(:status_notification) { Fabricate(:notification, account: local_follower, activity: status, type: :status) }
+    let!(:poll_notification) { Fabricate(:notification, account: local_follower, activity: poll, type: :poll) }
+    let!(:favourite_notification) { Fabricate(:notification, account: local_follower, activity: favourite, type: :favourite) }
+    let!(:follow_notification) { Fabricate(:notification, account: local_follower, activity: active_relationship, type: :follow) }
 
     subject do
       -> { described_class.new.call(account) }
     end
 
-    let!(:account) { Fabricate(:account) }
-    let!(:status) { Fabricate(:status, account: account) }
-    let!(:media_attachment) { Fabricate(:media_attachment, account: account) }
-    let!(:notification) { Fabricate(:notification, account: account) }
-    let!(:favourite) { Fabricate(:favourite, account: account) }
-    let!(:active_relationship) { Fabricate(:follow, account: account) }
-    let!(:passive_relationship) { Fabricate(:follow, target_account: account) }
-    let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', protocol: :activitypub) }
-    let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) }
-    let!(:endorsment) { Fabricate(:account_pin, account: passive_relationship.account, target_account: account) }
-
-    it 'deletes associated records' do
+    it 'deletes associated owned records' do
       is_expected.to change {
         [
           account.statuses,
@@ -31,54 +34,63 @@ RSpec.describe DeleteAccountService, type: :service do
           account.favourites,
           account.active_relationships,
           account.passive_relationships,
+          account.polls,
+        ].map(&:count)
+      }.from([2, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+    end
+
+    it 'deletes associated target records' do
+      is_expected.to change {
+        [
           AccountPin.where(target_account: account),
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+      }.from([1]).to([0])
     end
 
-    it 'sends a delete actor activity to all known inboxes' do
-      subject.call
-      expect(a_request(:post, "https://alice.com/inbox")).to have_been_made.once
-      expect(a_request(:post, "https://bob.com/inbox")).to have_been_made.once
+    it 'deletes associated target notifications' do
+      is_expected.to change {
+        [
+          'poll', 'favourite', 'status', 'mention', 'follow'
+        ].map { |type| Notification.where(type: type).count }
+      }.from([1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0])
     end
   end
 
-  describe '#call on remote account' do
+  describe '#call on local account' do
     before do
       stub_request(:post, "https://alice.com/inbox").to_return(status: 201)
       stub_request(:post, "https://bob.com/inbox").to_return(status: 201)
     end
 
-    subject do
-      -> { described_class.new.call(remote_bob) }
-    end
-
-    let!(:account) { Fabricate(:account) }
     let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', protocol: :activitypub) }
     let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) }
-    let!(:status) { Fabricate(:status, account: remote_bob) }
-    let!(:media_attachment) { Fabricate(:media_attachment, account: remote_bob) }
-    let!(:notification) { Fabricate(:notification, account: remote_bob) }
-    let!(:favourite) { Fabricate(:favourite, account: remote_bob) }
-    let!(:active_relationship) { Fabricate(:follow, account: remote_bob, target_account: account) }
-    let!(:passive_relationship) { Fabricate(:follow, target_account: remote_bob) }
 
-    it 'deletes associated records' do
-      is_expected.to change {
-        [
-          remote_bob.statuses,
-          remote_bob.media_attachments,
-          remote_bob.notifications,
-          remote_bob.favourites,
-          remote_bob.active_relationships,
-          remote_bob.passive_relationships,
-        ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0])
+    include_examples 'common behavior' do
+      let!(:account) { Fabricate(:account) }
+      let!(:local_follower) { Fabricate(:account) }
+
+      it 'sends a delete actor activity to all known inboxes' do
+        subject.call
+        expect(a_request(:post, "https://alice.com/inbox")).to have_been_made.once
+        expect(a_request(:post, "https://bob.com/inbox")).to have_been_made.once
+      end
+    end
+  end
+
+  describe '#call on remote account' do
+    before do
+      stub_request(:post, "https://alice.com/inbox").to_return(status: 201)
+      stub_request(:post, "https://bob.com/inbox").to_return(status: 201)
     end
 
-    it 'sends a reject follow to follwer inboxes' do
-      subject.call
-      expect(a_request(:post, remote_bob.inbox_url)).to have_been_made.once
+    include_examples 'common behavior' do
+      let!(:account) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) }
+      let!(:local_follower) { Fabricate(:account) }
+
+      it 'sends a reject follow to follwer inboxes' do
+        subject.call
+        expect(a_request(:post, account.inbox_url)).to have_been_made.once
+      end
     end
   end
 end
diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb
index 06676ec45..7ce75b2c7 100644
--- a/spec/services/remove_status_service_spec.rb
+++ b/spec/services/remove_status_service_spec.rb
@@ -3,7 +3,7 @@ require 'rails_helper'
 RSpec.describe RemoveStatusService, type: :service do
   subject { RemoveStatusService.new }
 
-  let!(:alice)  { Fabricate(:account) }
+  let!(:alice)  { Fabricate(:account, user: Fabricate(:user)) }
   let!(:bob)    { Fabricate(:account, username: 'bob', domain: 'example.com', salmon_url: 'http://example.com/salmon') }
   let!(:jeff)   { Fabricate(:account) }
   let!(:hank)   { Fabricate(:account, username: 'hank', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox') }
@@ -17,23 +17,33 @@ RSpec.describe RemoveStatusService, type: :service do
     hank.follow!(alice)
 
     @status = PostStatusService.new.call(alice, text: 'Hello @bob@example.com')
+    FavouriteService.new.call(jeff, @status)
     Fabricate(:status, account: bill, reblog: @status, uri: 'hoge')
-    subject.call(@status)
   end
 
   it 'removes status from author\'s home feed' do
+    subject.call(@status)
     expect(HomeFeed.new(alice).get(10)).to_not include(@status.id)
   end
 
   it 'removes status from local follower\'s home feed' do
+    subject.call(@status)
     expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id)
   end
 
   it 'sends delete activity to followers' do
+    subject.call(@status)
     expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.twice
   end
 
   it 'sends delete activity to rebloggers' do
+    subject.call(@status)
     expect(a_request(:post, 'http://example2.com/inbox')).to have_been_made
   end
+
+  it 'remove status from notifications' do
+    expect { subject.call(@status) }.to change {
+      Notification.where(activity_type: 'Favourite', from_account: jeff, account: alice).count
+    }.from(1).to(0)
+  end
 end
-- 
cgit 


From 9915d11c0d7a15b6775af8e78fcc4d836368f88d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 22 Dec 2020 17:13:55 +0100
Subject: Fix unnecessary queries when batch-removing statuses, 100x faster
 (#15387)

---
 app/models/favourite.rb                            |   2 +-
 app/models/status.rb                               |  12 +-
 app/services/batched_remove_status_service.rb      |  91 ++++++++-------
 app/services/delete_account_service.rb             | 126 ++++++++++++++++-----
 config/initializers/chewy.rb                       |   5 +
 lib/chewy/strategy/custom_sidekiq.rb               |  25 +---
 .../services/batched_remove_status_service_spec.rb |   5 +
 7 files changed, 167 insertions(+), 99 deletions(-)

(limited to 'app/models')

diff --git a/app/models/favourite.rb b/app/models/favourite.rb
index bf0ec4449..35028b7dd 100644
--- a/app/models/favourite.rb
+++ b/app/models/favourite.rb
@@ -36,7 +36,7 @@ class Favourite < ApplicationRecord
   end
 
   def decrement_cache_counters
-    return if association(:status).loaded? && (status.marked_for_destruction? || status.marked_for_mass_destruction?)
+    return if association(:status).loaded? && status.marked_for_destruction?
     status&.decrement_count!(:favourites_count)
   end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 96d90e1c2..b426f9d5b 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -228,14 +228,6 @@ class Status < ApplicationRecord
     @emojis = CustomEmoji.from_text(fields.join(' '), account.domain)
   end
 
-  def mark_for_mass_destruction!
-    @marked_for_mass_destruction = true
-  end
-
-  def marked_for_mass_destruction?
-    @marked_for_mass_destruction
-  end
-
   def replies_count
     status_stat&.replies_count || 0
   end
@@ -430,7 +422,7 @@ class Status < ApplicationRecord
   end
 
   def decrement_counter_caches
-    return if direct_visibility? || marked_for_mass_destruction?
+    return if direct_visibility?
 
     account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?
@@ -440,7 +432,7 @@ class Status < ApplicationRecord
   def unlink_from_conversations
     return unless direct_visibility?
 
-    mentioned_accounts = mentions.includes(:account).map(&:account)
+    mentioned_accounts = (association(:mentions).loaded? ? mentions : mentions.includes(:account)).map(&:account)
     inbox_owners       = mentioned_accounts.select(&:local?) + (account.local? ? [account] : [])
 
     inbox_owners.each do |inbox_owner|
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 28e5468b3..63ab89f2d 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -3,29 +3,45 @@
 class BatchedRemoveStatusService < BaseService
   include Redisable
 
-  # Delete given statuses and reblogs of them
-  # Remove statuses from home feeds
-  # Push delete events to streaming API for home feeds and public feeds
-  # @param [Enumerable<Status>] statuses A preferably batched array of statuses
+  # Delete multiple statuses and reblogs of them as efficiently as possible
+  # @param [Enumerable<Status>] statuses An array of statuses
   # @param [Hash] options
-  # @option [Boolean] :skip_side_effects
+  # @option [Boolean] :skip_side_effects Do not modify feeds and send updates to streaming API
   def call(statuses, **options)
-    statuses = Status.where(id: statuses.map(&:id)).includes(:account).flat_map { |status| [status] + status.reblogs.includes(:account).to_a }
+    ActiveRecord::Associations::Preloader.new.preload(statuses, options[:skip_side_effects] ? :reblogs : [:account, reblogs: :account])
 
-    @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a }
-    @tags     = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) }
+    statuses_and_reblogs = statuses.flat_map { |status| [status] + status.reblogs }
 
-    @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
+    # The conversations for direct visibility statuses also need
+    # to be manually updated. This part is not efficient but we
+    # rely on direct visibility statuses being relatively rare.
+    statuses_with_account_conversations = statuses.select(&:direct_visibility?)
 
-    statuses.each do |status|
-      status.mark_for_mass_destruction!
-      status.destroy
+    ActiveRecord::Associations::Preloader.new.preload(statuses_with_account_conversations, [mentions: :account])
+
+    statuses_with_account_conversations.each do |status|
+      status.send(:unlink_from_conversations)
     end
 
+    # We do not batch all deletes into one to avoid having a long-running
+    # transaction lock the database, but we use the delete method instead
+    # of destroy to avoid all callbacks. We rely on foreign keys to
+    # cascade the delete faster without loading the associations.
+    statuses_and_reblogs.each(&:delete)
+
+    # Since we skipped all callbacks, we also need to manually
+    # deindex the statuses
+    Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs)
+
     return if options[:skip_side_effects]
 
+    ActiveRecord::Associations::Preloader.new.preload(statuses_and_reblogs, :tags)
+
+    @tags          = statuses_and_reblogs.each_with_object({}) { |s, h| h[s.id] = s.tags.map { |tag| tag.name.mb_chars.downcase } }
+    @json_payloads = statuses_and_reblogs.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
+
     # Batch by source account
-    statuses.group_by(&:account_id).each_value do |account_statuses|
+    statuses_and_reblogs.group_by(&:account_id).each_value do |account_statuses|
       account = account_statuses.first.account
 
       next unless account
@@ -35,27 +51,31 @@ class BatchedRemoveStatusService < BaseService
     end
 
     # Cannot be batched
-    statuses.each do |status|
-      unpush_from_public_timelines(status)
+    redis.pipelined do
+      statuses_and_reblogs.each do |status|
+        unpush_from_public_timelines(status)
+      end
     end
   end
 
   private
 
   def unpush_from_home_timelines(account, statuses)
-    recipients = account.followers_for_local_distribution.to_a
-
-    recipients << account if account.local?
-
-    recipients.each do |follower|
+    account.followers_for_local_distribution.includes(:user).find_each do |follower|
       statuses.each do |status|
         FeedManager.instance.unpush_from_home(follower, status)
       end
     end
+
+    return unless account.local?
+
+    statuses.each do |status|
+      FeedManager.instance.unpush_from_home(account, status)
+    end
   end
 
   def unpush_from_list_timelines(account, statuses)
-    account.lists_for_local_distribution.select(:id, :account_id).each do |list|
+    account.lists_for_local_distribution.select(:id, :account_id).includes(account: :user).find_each do |list|
       statuses.each do |status|
         FeedManager.instance.unpush_from_list(list, status)
       end
@@ -67,26 +87,17 @@ class BatchedRemoveStatusService < BaseService
 
     payload = @json_payloads[status.id]
 
-    redis.pipelined do
-      redis.publish('timeline:public', payload)
-      if status.local?
-        redis.publish('timeline:public:local', payload)
-      else
-        redis.publish('timeline:public:remote', payload)
-      end
-      if status.media_attachments.any?
-        redis.publish('timeline:public:media', payload)
-        if status.local?
-          redis.publish('timeline:public:local:media', payload)
-        else
-          redis.publish('timeline:public:remote:media', payload)
-        end
-      end
+    redis.publish('timeline:public', payload)
+    redis.publish(status.local? ? 'timeline:public:local' : 'timeline:public:remote', payload)
 
-      @tags[status.id].each do |hashtag|
-        redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", payload)
-        redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", payload) if status.local?
-      end
+    if status.media_attachments.any?
+      redis.publish('timeline:public:media', payload)
+      redis.publish(status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', payload)
+    end
+
+    @tags[status.id].each do |hashtag|
+      redis.publish("timeline:hashtag:#{hashtag}", payload)
+      redis.publish("timeline:hashtag:#{hashtag}:local", payload) if status.local?
     end
   end
 end
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index fa834e775..5123a4697 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -6,15 +6,21 @@ class DeleteAccountService < BaseService
   ASSOCIATIONS_ON_SUSPEND = %w(
     account_pins
     active_relationships
+    aliases
     block_relationships
     blocked_by_relationships
+    bookmarks
     conversation_mutes
     conversations
     custom_filters
+    devices
     domain_blocks
     favourites
+    featured_tags
     follow_requests
+    identity_proofs
     list_accounts
+    migrations
     mute_relationships
     muted_by_relationships
     notifications
@@ -25,6 +31,29 @@ class DeleteAccountService < BaseService
     status_pins
   ).freeze
 
+  # The following associations have no important side-effects
+  # in callbacks and all of their own associations are secured
+  # by foreign keys, making them safe to delete without loading
+  # into memory
+  ASSOCIATIONS_WITHOUT_SIDE_EFFECTS = %w(
+    account_pins
+    aliases
+    conversation_mutes
+    conversations
+    custom_filters
+    devices
+    domain_blocks
+    featured_tags
+    follow_requests
+    identity_proofs
+    migrations
+    mute_relationships
+    muted_by_relationships
+    notifications
+    scheduled_statuses
+    status_pins
+  )
+
   ASSOCIATIONS_ON_DESTROY = %w(
     reports
     targeted_moderation_notes
@@ -55,19 +84,25 @@ class DeleteAccountService < BaseService
 
     @options[:skip_activitypub] = true if @options[:skip_side_effects]
 
-    reject_follows!
-    undo_follows!
-    purge_user!
-    purge_profile!
+    distribute_activities!
     purge_content!
     fulfill_deletion_request!
   end
 
   private
 
-  def reject_follows!
-    return if @account.local? || !@account.activitypub? || @options[:skip_activitypub]
+  def distribute_activities!
+    return if skip_activitypub?
 
+    if @account.local?
+      delete_actor!
+    elsif @account.activitypub?
+      reject_follows!
+      undo_follows!
+    end
+  end
+
+  def reject_follows!
     # When deleting a remote account, the account obviously doesn't
     # actually become deleted on its origin server, i.e. unlike a
     # locally deleted account it continues to have access to its home
@@ -81,8 +116,6 @@ class DeleteAccountService < BaseService
   end
 
   def undo_follows!
-    return if @account.local? || !@account.activitypub? || @options[:skip_activitypub]
-
     # When deleting a remote account, the account obviously doesn't
     # actually become deleted on its origin server, but following relationships
     # are severed on our end. Therefore, make the remote server aware that the
@@ -97,7 +130,7 @@ class DeleteAccountService < BaseService
   def purge_user!
     return if !@account.local? || @account.user.nil?
 
-    if @options[:reserve_email]
+    if keep_user_record?
       @account.user.disable!
       @account.user.invites.where(uses: 0).destroy_all
     else
@@ -106,34 +139,52 @@ class DeleteAccountService < BaseService
   end
 
   def purge_content!
-    distribute_delete_actor! if @account.local? && !@options[:skip_side_effects]
+    purge_user!
+    purge_profile!
+    purge_statuses!
+    purge_media_attachments!
+    purge_polls!
+    purge_generated_notifications!
+    purge_other_associations!
 
+    @account.destroy unless keep_account_record?
+  end
+
+  def purge_statuses!
     @account.statuses.reorder(nil).find_in_batches do |statuses|
-      statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username]
-      BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects])
+      statuses.reject! { |status| reported_status_ids.include?(status.id) } if keep_account_record?
+
+      BatchedRemoveStatusService.new.call(statuses, skip_side_effects: skip_side_effects?)
     end
+  end
 
+  def purge_media_attachments!
     @account.media_attachments.reorder(nil).find_each do |media_attachment|
-      next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id)
+      next if keep_account_record? && reported_status_ids.include?(media_attachment.status_id)
 
       media_attachment.destroy
     end
+  end
 
+  def purge_polls!
     @account.polls.reorder(nil).find_each do |poll|
-      next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id)
+      next if keep_account_record? && reported_status_ids.include?(poll.status_id)
 
-      # We can safely delete the poll rather than destroy it, as any non-reported
-      # status should have been deleted already, as long as we take care of
-      # notifications.
-      Notification.where(poll: poll).delete_all
       poll.delete
     end
+  end
 
+  def purge_generated_notifications!
+    # By deleting polls and statuses without callbacks, we've left behind
+    # polymorphically associated notifications generated by this account
+
+    Notification.where(from_account: @account).in_batches.delete_all
+  end
+
+  def purge_other_associations!
     associations_for_destruction.each do |association_name|
-      destroy_all(@account.public_send(association_name))
+      purge_association(association_name)
     end
-
-    @account.destroy unless @options[:reserve_username]
   end
 
   def purge_profile!
@@ -141,7 +192,7 @@ class DeleteAccountService < BaseService
     # there is no point wasting time updating
     # its values first
 
-    return unless @options[:reserve_username]
+    return unless keep_account_record?
 
     @account.silenced_at       = nil
     @account.suspended_at      = @options[:suspended_at] || Time.now.utc
@@ -156,6 +207,7 @@ class DeleteAccountService < BaseService
     @account.followers_count   = 0
     @account.following_count   = 0
     @account.moved_to_account  = nil
+    @account.also_known_as     = []
     @account.trust_level       = :untrusted
     @account.avatar.destroy
     @account.header.destroy
@@ -166,11 +218,17 @@ class DeleteAccountService < BaseService
     @account.deletion_request&.destroy
   end
 
-  def destroy_all(association)
-    association.in_batches.destroy_all
+  def purge_association(association_name)
+    association = @account.public_send(association_name)
+
+    if ASSOCIATIONS_WITHOUT_SIDE_EFFECTS.include?(association_name)
+      association.in_batches.delete_all
+    else
+      association.in_batches.destroy_all
+    end
   end
 
-  def distribute_delete_actor!
+  def delete_actor!
     ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url|
       [delete_actor_json, @account.id, inbox_url]
     end
@@ -197,10 +255,26 @@ class DeleteAccountService < BaseService
   end
 
   def associations_for_destruction
-    if @options[:reserve_username]
+    if keep_account_record?
       ASSOCIATIONS_ON_SUSPEND
     else
       ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY
     end
   end
+
+  def keep_user_record?
+    @options[:reserve_email]
+  end
+
+  def keep_account_record?
+    @options[:reserve_username]
+  end
+
+  def skip_side_effects?
+    @options[:skip_side_effects]
+  end
+
+  def skip_activitypub?
+    @options[:skip_activitypub]
+  end
 end
diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb
index 8f54abf77..9fc9b2f1a 100644
--- a/config/initializers/chewy.rb
+++ b/config/initializers/chewy.rb
@@ -12,6 +12,10 @@ Chewy.settings = {
   sidekiq: { queue: 'pull' },
 }
 
+# We use our own async strategy even outside the request-response
+# cycle, which takes care of checking if ElasticSearch is enabled
+# or not. However, mind that for the Rails console, the :urgent
+# strategy is set automatically with no way to override it.
 Chewy.root_strategy              = :custom_sidekiq
 Chewy.request_strategy           = :custom_sidekiq
 Chewy.use_after_commit_callbacks = false
@@ -37,6 +41,7 @@ Elasticsearch::Transport::Client.prepend Module.new {
     super arguments
   end
 }
+
 Elasticsearch::API::Indices::IndicesClient.prepend Module.new {
   def create(arguments = {})
     arguments[:include_type_name] = true
diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb
index 3e54326ba..794ae4ed4 100644
--- a/lib/chewy/strategy/custom_sidekiq.rb
+++ b/lib/chewy/strategy/custom_sidekiq.rb
@@ -2,29 +2,10 @@
 
 module Chewy
   class Strategy
-    class CustomSidekiq < Base
-      class Worker
-        include ::Sidekiq::Worker
-
-        sidekiq_options queue: 'pull'
-
-        def perform(type, ids, options = {})
-          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
-          type.constantize.import!(ids, options)
-        end
-      end
-
-      def update(type, objects, _options = {})
-        return unless Chewy.enabled?
-
-        ids = type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
-
-        return if ids.empty?
-
-        Worker.perform_async(type.name, ids)
+    class CustomSidekiq < Sidekiq
+      def update(_type, _objects, _options = {})
+        super if Chewy.enabled?
       end
-
-      def leave; end
     end
   end
 end
diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb
index f84256f18..239859f06 100644
--- a/spec/services/batched_remove_status_service_spec.rb
+++ b/spec/services/batched_remove_status_service_spec.rb
@@ -26,6 +26,11 @@ RSpec.describe BatchedRemoveStatusService, type: :service do
     subject.call([status1, status2])
   end
 
+  it 'removes statuses' do
+    expect { Status.find(status1.id) }.to raise_error ActiveRecord::RecordNotFound
+    expect { Status.find(status2.id) }.to raise_error ActiveRecord::RecordNotFound
+  end
+
   it 'removes statuses from author\'s home feed' do
     expect(HomeFeed.new(alice).get(10)).to_not include([status1.id, status2.id])
   end
-- 
cgit 


From 1cf2c3a810bba937c7702c342a3ff37c3d37391a Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 22 Dec 2020 17:14:32 +0100
Subject: Fix external user creation failing when invite request text is
 required (#15405)

* Fix external user creation failing when invite request text is required

Also fixes tootctl-based user creation.

* Add test about invites when invite request text is otherwise required

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/models/user.rb                                     | 12 ++++++++++--
 lib/mastodon/accounts_cli.rb                           |  2 +-
 lib/tasks/mastodon.rake                                |  2 +-
 spec/controllers/auth/registrations_controller_spec.rb |  5 ++++-
 4 files changed, 16 insertions(+), 5 deletions(-)

(limited to 'app/models')

diff --git a/app/models/user.rb b/app/models/user.rb
index dd96bbf8c..f8c8a6ab5 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -83,7 +83,7 @@ class User < ApplicationRecord
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text }
-  validates :invite_request, presence: true, on: :create, if: -> { Setting.require_invite_text && !invited? }
+  validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
   validates_with BlacklistedEmailValidator, on: :create
@@ -128,7 +128,7 @@ class User < ApplicationRecord
            to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code, :sign_in_token_attempt
-  attr_writer :external
+  attr_writer :external, :bypass_invite_request_check
 
   def confirmed?
     confirmed_at.present?
@@ -429,6 +429,10 @@ class User < ApplicationRecord
     !!@external
   end
 
+  def bypass_invite_request_check?
+    @bypass_invite_request_check
+  end
+
   def sanitize_languages
     return if chosen_languages.nil?
     chosen_languages.reject!(&:blank?)
@@ -466,4 +470,8 @@ class User < ApplicationRecord
   def validate_email_dns?
     email_changed? && !(Rails.env.test? || Rails.env.development?)
   end
+
+  def invite_text_required?
+    Setting.require_invite_text && !invited? && !external? && !bypass_invite_request_check?
+  end
 end
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
index 5275f04cf..cdd1db995 100644
--- a/lib/mastodon/accounts_cli.rb
+++ b/lib/mastodon/accounts_cli.rb
@@ -77,7 +77,7 @@ module Mastodon
     def create(username)
       account  = Account.new(username: username)
       password = SecureRandom.hex
-      user     = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil)
+      user     = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true)
 
       if options[:reattach]
         account = Account.find_local(username) || Account.new(username: username)
diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index 9e80989ef..2ad1e778b 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -412,7 +412,7 @@ namespace :mastodon do
 
           password = SecureRandom.hex(16)
 
-          user = User.new(admin: true, email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username })
+          user = User.new(admin: true, email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true)
           user.save(validate: false)
 
           prompt.ok "You can login with the password: #{password}"
diff --git a/spec/controllers/auth/registrations_controller_spec.rb b/spec/controllers/auth/registrations_controller_spec.rb
index c701a3b8b..ccf304a93 100644
--- a/spec/controllers/auth/registrations_controller_spec.rb
+++ b/spec/controllers/auth/registrations_controller_spec.rb
@@ -195,16 +195,19 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
       end
     end
 
-    context 'approval-based registrations with valid invite' do
+    context 'approval-based registrations with valid invite and required invite text' do
       around do |example|
         registrations_mode = Setting.registrations_mode
+        require_invite_text = Setting.require_invite_text
         example.run
+        Setting.require_invite_text = require_invite_text
         Setting.registrations_mode = registrations_mode
       end
 
       subject do
         inviter = Fabricate(:user, confirmed_at: 2.days.ago)
         Setting.registrations_mode = 'approved'
+        Setting.require_invite_text = true
         request.headers["Accept-Language"] = accept_language
         invite = Fabricate(:invite, user: inviter, max_uses: nil, expires_at: 1.hour.from_now)
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678', 'invite_code': invite.code, agreement: 'true' } }
-- 
cgit 


From 3249d35bdcd9a495af3277dfb4b2129d7ef80f15 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 22 Dec 2020 23:57:46 +0100
Subject: Improve account deletion performances further (#15407)

* Delete status records by batches of 50

* Do not precompute values that are only used once

* Do not generate redis events for removal of public toots older than two weeks

* Filter reported toots a priori for polls and status deletion

* Do not process reblogs when cleaning up public timelines

As in Mastodon proper, reblogs don't appear in public TLs

* Clean the deleted account's own feed in one go

* Refactor Account#clean_feed_manager and List#clean_feed_manager

* Delete instead of destroy a few more associations

* Fix preloading

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/lib/feed_manager.rb                            | 30 ++++++++++++++++++++++
 app/models/account.rb                              | 13 +---------
 app/models/list.rb                                 | 13 +---------
 app/services/batched_remove_status_service.rb      | 24 +++++------------
 app/services/delete_account_service.rb             | 20 +++++++++------
 app/workers/scheduler/feed_cleanup_scheduler.rb    | 30 ++--------------------
 .../services/batched_remove_status_service_spec.rb |  4 ---
 7 files changed, 53 insertions(+), 81 deletions(-)

(limited to 'app/models')

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 5e01ef67a..f0ad3e21f 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -230,6 +230,36 @@ class FeedManager
     end
   end
 
+  # Completely clear multiple feeds at once
+  # @param [Symbol] type
+  # @param [Array<Integer>] ids
+  # @return [void]
+  def clean_feeds!(type, ids)
+    reblogged_id_sets = {}
+
+    redis.pipelined do
+      ids.each do |feed_id|
+        redis.del(key(type, feed_id))
+        reblog_key = key(type, feed_id, 'reblogs')
+        # We collect a future for this: we don't block while getting
+        # it, but we can iterate over it later.
+        reblogged_id_sets[feed_id] = redis.zrange(reblog_key, 0, -1)
+        redis.del(reblog_key)
+      end
+    end
+
+    # Remove all of the reblog tracking keys we just removed the
+    # references to.
+    redis.pipelined do
+      reblogged_id_sets.each do |feed_id, future|
+        future.value.each do |reblogged_id|
+          reblog_set_key = key(type, feed_id, "reblogs:#{reblogged_id}")
+          redis.del(reblog_set_key)
+        end
+      end
+    end
+  end
+
   private
 
   # Trim a feed to maximum size by removing older items
diff --git a/app/models/account.rb b/app/models/account.rb
index 80eb92a71..e6cf03fa8 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -578,17 +578,6 @@ class Account < ApplicationRecord
   end
 
   def clean_feed_manager
-    reblog_key       = FeedManager.instance.key(:home, id, 'reblogs')
-    reblogged_id_set = Redis.current.zrange(reblog_key, 0, -1)
-
-    Redis.current.pipelined do
-      Redis.current.del(FeedManager.instance.key(:home, id))
-      Redis.current.del(reblog_key)
-
-      reblogged_id_set.each do |reblogged_id|
-        reblog_set_key = FeedManager.instance.key(:home, id, "reblogs:#{reblogged_id}")
-        Redis.current.del(reblog_set_key)
-      end
-    end
+    FeedManager.instance.clean_feeds!(:home, [id])
   end
 end
diff --git a/app/models/list.rb b/app/models/list.rb
index 655d55ff6..cdc6ebdb3 100644
--- a/app/models/list.rb
+++ b/app/models/list.rb
@@ -34,17 +34,6 @@ class List < ApplicationRecord
   private
 
   def clean_feed_manager
-    reblog_key       = FeedManager.instance.key(:list, id, 'reblogs')
-    reblogged_id_set = Redis.current.zrange(reblog_key, 0, -1)
-
-    Redis.current.pipelined do
-      Redis.current.del(FeedManager.instance.key(:list, id))
-      Redis.current.del(reblog_key)
-
-      reblogged_id_set.each do |reblogged_id|
-        reblog_set_key = FeedManager.instance.key(:list, id, "reblogs:#{reblogged_id}")
-        Redis.current.del(reblog_set_key)
-      end
-    end
+    FeedManager.instance.clean_feeds!(:list, [id])
   end
 end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 3ec000110..61617d958 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -8,7 +8,7 @@ class BatchedRemoveStatusService < BaseService
   # @param [Hash] options
   # @option [Boolean] :skip_side_effects Do not modify feeds and send updates to streaming API
   def call(statuses, **options)
-    ActiveRecord::Associations::Preloader.new.preload(statuses, options[:skip_side_effects] ? :reblogs : [:account, reblogs: :account])
+    ActiveRecord::Associations::Preloader.new.preload(statuses, options[:skip_side_effects] ? :reblogs : [:account, :tags, reblogs: :account])
 
     statuses_and_reblogs = statuses.flat_map { |status| [status] + status.reblogs }
 
@@ -27,7 +27,7 @@ class BatchedRemoveStatusService < BaseService
     # transaction lock the database, but we use the delete method instead
     # of destroy to avoid all callbacks. We rely on foreign keys to
     # cascade the delete faster without loading the associations.
-    statuses_and_reblogs.each(&:delete)
+    statuses_and_reblogs.each_slice(50) { |slice| Status.where(id: slice.map(&:id)).delete_all }
 
     # Since we skipped all callbacks, we also need to manually
     # deindex the statuses
@@ -35,11 +35,6 @@ class BatchedRemoveStatusService < BaseService
 
     return if options[:skip_side_effects]
 
-    ActiveRecord::Associations::Preloader.new.preload(statuses_and_reblogs, :tags)
-
-    @tags          = statuses_and_reblogs.each_with_object({}) { |s, h| h[s.id] = s.tags.map { |tag| tag.name.mb_chars.downcase } }
-    @json_payloads = statuses_and_reblogs.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
-
     # Batch by source account
     statuses_and_reblogs.group_by(&:account_id).each_value do |account_statuses|
       account = account_statuses.first.account
@@ -51,8 +46,9 @@ class BatchedRemoveStatusService < BaseService
     end
 
     # Cannot be batched
+    @status_id_cutoff = Mastodon::Snowflake.id_at(2.weeks.ago)
     redis.pipelined do
-      statuses_and_reblogs.each do |status|
+      statuses.each do |status|
         unpush_from_public_timelines(status)
       end
     end
@@ -66,12 +62,6 @@ class BatchedRemoveStatusService < BaseService
         FeedManager.instance.unpush_from_home(follower, status)
       end
     end
-
-    return unless account.local?
-
-    statuses.each do |status|
-      FeedManager.instance.unpush_from_home(account, status)
-    end
   end
 
   def unpush_from_list_timelines(account, statuses)
@@ -83,9 +73,9 @@ class BatchedRemoveStatusService < BaseService
   end
 
   def unpush_from_public_timelines(status)
-    return unless status.public_visibility?
+    return unless status.public_visibility? && status.id > @status_id_cutoff
 
-    payload = @json_payloads[status.id]
+    payload = Oj.dump(event: :delete, payload: status.id.to_s)
 
     redis.publish('timeline:public', payload)
     redis.publish(status.local? ? 'timeline:public:local' : 'timeline:public:remote', payload)
@@ -95,7 +85,7 @@ class BatchedRemoveStatusService < BaseService
       redis.publish(status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', payload)
     end
 
-    @tags[status.id].each do |hashtag|
+    status.tags.map { |tag| tag.name.mb_chars.downcase }.each do |hashtag|
       redis.publish("timeline:hashtag:#{hashtag}", payload)
       redis.publish("timeline:hashtag:#{hashtag}:local", payload) if status.local?
     end
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index 5123a4697..58f6ef2ab 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -46,10 +46,12 @@ class DeleteAccountService < BaseService
     featured_tags
     follow_requests
     identity_proofs
+    list_accounts
     migrations
     mute_relationships
     muted_by_relationships
     notifications
+    owned_lists
     scheduled_statuses
     status_pins
   )
@@ -145,15 +147,14 @@ class DeleteAccountService < BaseService
     purge_media_attachments!
     purge_polls!
     purge_generated_notifications!
+    purge_feeds!
     purge_other_associations!
 
     @account.destroy unless keep_account_record?
   end
 
   def purge_statuses!
-    @account.statuses.reorder(nil).find_in_batches do |statuses|
-      statuses.reject! { |status| reported_status_ids.include?(status.id) } if keep_account_record?
-
+    @account.statuses.reorder(nil).where.not(id: reported_status_ids).in_batches do |statuses|
       BatchedRemoveStatusService.new.call(statuses, skip_side_effects: skip_side_effects?)
     end
   end
@@ -167,11 +168,7 @@ class DeleteAccountService < BaseService
   end
 
   def purge_polls!
-    @account.polls.reorder(nil).find_each do |poll|
-      next if keep_account_record? && reported_status_ids.include?(poll.status_id)
-
-      poll.delete
-    end
+    @account.polls.reorder(nil).where.not(status_id: reported_status_ids).in_batches.delete_all
   end
 
   def purge_generated_notifications!
@@ -187,6 +184,13 @@ class DeleteAccountService < BaseService
     end
   end
 
+  def purge_feeds!
+    return unless @account.local?
+
+    FeedManager.instance.clean_feeds!(:home, [@account.id])
+    FeedManager.instance.clean_feeds!(:list, @account.owned_lists.pluck(:id))
+  end
+
   def purge_profile!
     # If the account is going to be destroyed
     # there is no point wasting time updating
diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb
index 458fe6193..42b29f4ec 100644
--- a/app/workers/scheduler/feed_cleanup_scheduler.rb
+++ b/app/workers/scheduler/feed_cleanup_scheduler.rb
@@ -14,37 +14,11 @@ class Scheduler::FeedCleanupScheduler
   private
 
   def clean_home_feeds!
-    clean_feeds!(inactive_account_ids, :home)
+    feed_manager.clean_feeds!(:home, inactive_account_ids)
   end
 
   def clean_list_feeds!
-    clean_feeds!(inactive_list_ids, :list)
-  end
-
-  def clean_feeds!(ids, type)
-    reblogged_id_sets = {}
-
-    redis.pipelined do
-      ids.each do |feed_id|
-        redis.del(feed_manager.key(type, feed_id))
-        reblog_key = feed_manager.key(type, feed_id, 'reblogs')
-        # We collect a future for this: we don't block while getting
-        # it, but we can iterate over it later.
-        reblogged_id_sets[feed_id] = redis.zrange(reblog_key, 0, -1)
-        redis.del(reblog_key)
-      end
-    end
-
-    # Remove all of the reblog tracking keys we just removed the
-    # references to.
-    redis.pipelined do
-      reblogged_id_sets.each do |feed_id, future|
-        future.value.each do |reblogged_id|
-          reblog_set_key = feed_manager.key(type, feed_id, "reblogs:#{reblogged_id}")
-          redis.del(reblog_set_key)
-        end
-      end
-    end
+    feed_manager.clean_feeds!(:list, inactive_list_ids)
   end
 
   def inactive_account_ids
diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb
index 239859f06..c1f54a6fd 100644
--- a/spec/services/batched_remove_status_service_spec.rb
+++ b/spec/services/batched_remove_status_service_spec.rb
@@ -43,10 +43,6 @@ RSpec.describe BatchedRemoveStatusService, type: :service do
     expect(Redis.current).to have_received(:publish).with("timeline:#{jeff.id}", any_args).at_least(:once)
   end
 
-  it 'notifies streaming API of author' do
-    expect(Redis.current).to have_received(:publish).with("timeline:#{alice.id}", any_args).at_least(:once)
-  end
-
   it 'notifies streaming API of public timeline' do
     expect(Redis.current).to have_received(:publish).with('timeline:public', any_args).at_least(:once)
   end
-- 
cgit