Fix performance of account timelines (#17709)

* Fix performance of account timelines

* Various fixes and improvements

* Fix duplicate results being returned

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

* Fix grouping for pinned statuses scope

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

4 files changed, 137 insertions, 62 deletions

diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb
index b2aab56a5..cd3992502 100644
--- a/app/controllers/activitypub/outboxes_controller.rb
+++ b/app/controllers/activitypub/outboxes_controller.rb
@@ -62,7 +62,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
     return unless page_requested?
 
     @statuses = cache_collection_paginated_by_id(
-      @account.statuses.permitted_for(@account, signed_request_account),
+      AccountStatusesFilter.new(@account, signed_request_account).results,
       Status,
       LIMIT,
       params_slice(:max_id, :min_id, :since_id)
diff --git a/app/controllers/api/v1/accounts/statuses_controller.rb b/app/controllers/api/v1/accounts/statuses_controller.rb
index 2c027ea76..38c9f5a20 100644
--- a/app/controllers/api/v1/accounts/statuses_controller.rb
+++ b/app/controllers/api/v1/accounts/statuses_controller.rb
@@ -22,53 +22,16 @@ class Api::V1::Accounts::StatusesController < Api::BaseController
   end
 
   def cached_account_statuses
-    statuses = truthy_param?(:pinned) ? pinned_scope : permitted_account_statuses
-
-    statuses.merge!(only_media_scope) if truthy_param?(:only_media)
-    statuses.merge!(no_replies_scope) if truthy_param?(:exclude_replies)
-    statuses.merge!(no_reblogs_scope) if truthy_param?(:exclude_reblogs)
-    statuses.merge!(hashtag_scope)    if params[:tagged].present?
-
     cache_collection_paginated_by_id(
-      statuses,
+      AccountStatusesFilter.new(@account, current_account, params).results,
       Status,
       limit_param(DEFAULT_STATUSES_LIMIT),
       params_slice(:max_id, :since_id, :min_id)
     )
   end
 
-  def permitted_account_statuses
-    @account.statuses.permitted_for(@account, current_account)
-  end
-
-  def only_media_scope
-    Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)
-  end
-
-  def pinned_scope
-    @account.pinned_statuses.permitted_for(@account, current_account)
-  end
-
-  def no_replies_scope
-    Status.without_replies
-  end
-
-  def no_reblogs_scope
-    Status.without_reblogs
-  end
-
-  def hashtag_scope
-    tag = Tag.find_normalized(params[:tagged])
-
-    if tag
-      Status.tagged_with(tag.id)
-    else
-      Status.none
-    end
-  end
-
   def pagination_params(core_params)
-    params.slice(:limit, :only_media, :exclude_replies).permit(:limit, :only_media, :exclude_replies).merge(core_params)
+    params.slice(:limit, *AccountStatusesFilter::KEYS).permit(:limit, *AccountStatusesFilter::KEYS).merge(core_params)
   end
 
   def insert_pagination_headers
diff --git a/app/models/account_statuses_filter.rb b/app/models/account_statuses_filter.rb
new file mode 100644
index 000000000..211f41478
--- /dev/null
+++ b/app/models/account_statuses_filter.rb
@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+class AccountStatusesFilter
+  KEYS = %i(
+    pinned
+    tagged
+    only_media
+    exclude_replies
+    exclude_reblogs
+  ).freeze
+
+  attr_reader :params, :account, :current_account
+
+  def initialize(account, current_account, params = {})
+    @account         = account
+    @current_account = current_account
+    @params          = params
+  end
+
+  def results
+    scope = initial_scope
+
+    scope.merge!(pinned_scope)     if pinned?
+    scope.merge!(only_media_scope) if only_media?
+    scope.merge!(no_replies_scope) if exclude_replies?
+    scope.merge!(no_reblogs_scope) if exclude_reblogs?
+    scope.merge!(hashtag_scope)    if tagged?
+
+    scope
+  end
+
+  private
+
+  def initial_scope
+    if suspended?
+      Status.none
+    elsif anonymous?
+      account.statuses.where(visibility: %i(public unlisted))
+    elsif author?
+      account.statuses.all # NOTE: #merge! does not work without the #all
+    elsif blocked?
+      Status.none
+    else
+      filtered_scope
+    end
+  end
+
+  def filtered_scope
+    scope = account.statuses.left_outer_joins(:mentions)
+
+    scope.merge!(scope.where(visibility: follower? ? %i(public unlisted private) : %i(public unlisted)).or(scope.where(mentions: { account_id: current_account.id })).group(Status.arel_table[:id]))
+    scope.merge!(filtered_reblogs_scope) if reblogs_may_occur?
+
+    scope
+  end
+
+  def filtered_reblogs_scope
+    Status.left_outer_joins(:reblog).where(reblog_of_id: nil).or(Status.where.not(reblogs_statuses: { account_id: current_account.excluded_from_timeline_account_ids }))
+  end
+
+  def only_media_scope
+    Status.joins(:media_attachments).merge(account.media_attachments.reorder(nil)).group(Status.arel_table[:id])
+  end
+
+  def no_replies_scope
+    Status.without_replies
+  end
+
+  def no_reblogs_scope
+    Status.without_reblogs
+  end
+
+  def pinned_scope
+    account.pinned_statuses.group(Status.arel_table[:id], StatusPin.arel_table[:created_at])
+  end
+
+  def hashtag_scope
+    tag = Tag.find_normalized(params[:tagged])
+
+    if tag
+      Status.tagged_with(tag.id)
+    else
+      Status.none
+    end
+  end
+
+  def suspended?
+    account.suspended?
+  end
+
+  def anonymous?
+    current_account.nil?
+  end
+
+  def author?
+    current_account.id == account.id
+  end
+
+  def blocked?
+    account.blocking?(current_account) || (current_account.domain.present? && account.domain_blocking?(current_account.domain))
+  end
+
+  def follower?
+    current_account.following?(account)
+  end
+
+  def reblogs_may_occur?
+    !exclude_reblogs? && !only_media? && !tagged?
+  end
+
+  def pinned?
+    truthy_param?(:pinned)
+  end
+
+  def only_media?
+    truthy_param?(:only_media)
+  end
+
+  def exclude_replies?
+    truthy_param?(:exclude_replies)
+  end
+
+  def exclude_reblogs?
+    truthy_param?(:exclude_reblogs)
+  end
+
+  def tagged?
+    params[:tagged].present?
+  end
+
+  def truthy_param?(key)
+    ActiveModel::Type::Boolean.new.cast(params[key])
+  end
+end
diff --git a/app/models/status.rb b/app/models/status.rb
index 60dde5045..af3e645dc 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -345,28 +345,6 @@ class Status < ApplicationRecord
       end
     end
 
-    def permitted_for(target_account, account)
-      visibility = [:public, :unlisted]
-
-      if account.nil?
-        where(visibility: visibility)
-      elsif target_account.blocking?(account) || (account.domain.present? && target_account.domain_blocking?(account.domain)) # get rid of blocked peeps
-        none
-      elsif account.id == target_account.id # author can see own stuff
-        all
-      else
-        # followers can see followers-only stuff, but also things they are mentioned in.
-        # non-followers can see everything that isn't private/direct, but can see stuff they are mentioned in.
-        visibility.push(:private) if account.following?(target_account)
-
-        scope = left_outer_joins(:reblog)
-
-        scope.where(visibility: visibility)
-             .or(scope.where(id: account.mentions.select(:status_id)))
-             .merge(scope.where(reblog_of_id: nil).or(scope.where.not(reblogs_statuses: { account_id: account.excluded_from_timeline_account_ids })))
-      end
-    end
-
     def from_text(text)
       return [] if text.blank?