Add cold-start follow recommendations (#15945)

20 files changed, 398 insertions, 20 deletions

diff --git a/app/controllers/admin/follow_recommendations_controller.rb b/app/controllers/admin/follow_recommendations_controller.rb
new file mode 100644
index 000000000..e3eac62b3
--- /dev/null
+++ b/app/controllers/admin/follow_recommendations_controller.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Admin
+  class FollowRecommendationsController < BaseController
+    before_action :set_language
+
+    def show
+      authorize :follow_recommendation, :show?
+
+      @form     = Form::AccountBatch.new
+      @accounts = filtered_follow_recommendations
+    end
+
+    def update
+      @form = Form::AccountBatch.new(form_account_batch_params.merge(current_account: current_account, action: action_from_button))
+      @form.save
+    rescue ActionController::ParameterMissing
+      # Do nothing
+    ensure
+      redirect_to admin_follow_recommendations_path(filter_params)
+    end
+
+    private
+
+    def set_language
+      @language = follow_recommendation_filter.language
+    end
+
+    def filtered_follow_recommendations
+      follow_recommendation_filter.results
+    end
+
+    def follow_recommendation_filter
+      @follow_recommendation_filter ||= FollowRecommendationFilter.new(filter_params)
+    end
+
+    def form_account_batch_params
+      params.require(:form_account_batch).permit(:action, account_ids: [])
+    end
+
+    def filter_params
+      params.slice(*FollowRecommendationFilter::KEYS).permit(*FollowRecommendationFilter::KEYS)
+    end
+
+    def action_from_button
+      if params[:suppress]
+        'suppress_follow_recommendation'
+      elsif params[:unsuppress]
+        'unsuppress_follow_recommendation'
+      end
+    end
+  end
+end
diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb
index 52054160d..b2788cc76 100644
--- a/app/controllers/api/v1/suggestions_controller.rb
+++ b/app/controllers/api/v1/suggestions_controller.rb
@@ -19,6 +19,6 @@ class Api::V1::SuggestionsController < Api::BaseController
   private
 
   def set_accounts
-    @accounts = PotentialFriendshipTracker.get(current_account.id, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT))
+    @accounts = PotentialFriendshipTracker.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT))
   end
 end
diff --git a/app/controllers/api/v2/suggestions_controller.rb b/app/controllers/api/v2/suggestions_controller.rb
new file mode 100644
index 000000000..35eb276c0
--- /dev/null
+++ b/app/controllers/api/v2/suggestions_controller.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class Api::V2::SuggestionsController < Api::BaseController
+  include Authorization
+
+  before_action -> { doorkeeper_authorize! :read }
+  before_action :require_user!
+  before_action :set_suggestions
+
+  def index
+    render json: @suggestions, each_serializer: REST::SuggestionSerializer
+  end
+
+  private
+
+  def set_suggestions
+    @suggestions = AccountSuggestions.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT))
+  end
+end
diff --git a/app/javascript/mastodon/actions/suggestions.js b/app/javascript/mastodon/actions/suggestions.js
index b15bd916b..0bf959017 100644
--- a/app/javascript/mastodon/actions/suggestions.js
+++ b/app/javascript/mastodon/actions/suggestions.js
@@ -11,8 +11,8 @@ export function fetchSuggestions() {
   return (dispatch, getState) => {
     dispatch(fetchSuggestionsRequest());
 
-    api(getState).get('/api/v1/suggestions').then(response => {
-      dispatch(importFetchedAccounts(response.data));
+    api(getState).get('/api/v2/suggestions').then(response => {
+      dispatch(importFetchedAccounts(response.data.map(x => x.account)));
       dispatch(fetchSuggestionsSuccess(response.data));
     }).catch(error => dispatch(fetchSuggestionsFail(error)));
   };
@@ -25,10 +25,10 @@ export function fetchSuggestionsRequest() {
   };
 };
 
-export function fetchSuggestionsSuccess(accounts) {
+export function fetchSuggestionsSuccess(suggestions) {
   return {
     type: SUGGESTIONS_FETCH_SUCCESS,
-    accounts,
+    suggestions,
     skipLoading: true,
   };
 };
diff --git a/app/javascript/mastodon/features/compose/components/search_results.js b/app/javascript/mastodon/features/compose/components/search_results.js
index 4b4cdff74..c4e160b8a 100644
--- a/app/javascript/mastodon/features/compose/components/search_results.js
+++ b/app/javascript/mastodon/features/compose/components/search_results.js
@@ -51,13 +51,13 @@ class SearchResults extends ImmutablePureComponent {
               <FormattedMessage id='suggestions.header' defaultMessage='You might be interested in…' />
             </div>
 
-            {suggestions && suggestions.map(accountId => (
+            {suggestions && suggestions.map(suggestion => (
               <AccountContainer
-                key={accountId}
-                id={accountId}
-                actionIcon='times'
-                actionTitle={intl.formatMessage(messages.dismissSuggestion)}
-                onActionClick={dismissSuggestion}
+                key={suggestion.get('account')}
+                id={suggestion.get('account')}
+                actionIcon={suggestion.get('source') === 'past_interaction' && 'times'}
+                actionTitle={suggestion.get('source') === 'past_interaction' && intl.formatMessage(messages.dismissSuggestion)}
+                onActionClick={suggestion.get('source') === 'past_interaction' && dismissSuggestion}
               />
             ))}
           </div>
diff --git a/app/javascript/mastodon/reducers/suggestions.js b/app/javascript/mastodon/reducers/suggestions.js
index 834be728f..1a6e66ee7 100644
--- a/app/javascript/mastodon/reducers/suggestions.js
+++ b/app/javascript/mastodon/reducers/suggestions.js
@@ -19,18 +19,18 @@ export default function suggestionsReducer(state = initialState, action) {
     return state.set('isLoading', true);
   case SUGGESTIONS_FETCH_SUCCESS:
     return state.withMutations(map => {
-      map.set('items', fromJS(action.accounts.map(x => x.id)));
+      map.set('items', fromJS(action.suggestions.map(x => ({ ...x, account: x.account.id }))));
       map.set('isLoading', false);
     });
   case SUGGESTIONS_FETCH_FAIL:
     return state.set('isLoading', false);
   case SUGGESTIONS_DISMISS:
-    return state.update('items', list => list.filterNot(id => id === action.id));
+    return state.update('items', list => list.filterNot(x => x.account === action.id));
   case ACCOUNT_BLOCK_SUCCESS:
   case ACCOUNT_MUTE_SUCCESS:
-    return state.update('items', list => list.filterNot(id => id === action.relationship.id));
+    return state.update('items', list => list.filterNot(x => x.account === action.relationship.id));
   case DOMAIN_BLOCK_SUCCESS:
-    return state.update('items', list => list.filterNot(id => action.accounts.includes(id)));
+    return state.update('items', list => list.filterNot(x => action.accounts.includes(x.account)));
   default:
     return state;
   }
diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb
index 188aa4a27..e72d454b6 100644
--- a/app/lib/potential_friendship_tracker.rb
+++ b/app/lib/potential_friendship_tracker.rb
@@ -28,10 +28,14 @@ class PotentialFriendshipTracker
       redis.zrem("interactions:#{account_id}", target_account_id)
     end
 
-    def get(account_id, limit: 20, offset: 0)
-      account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit)
-      return [] if account_ids.empty?
-      Account.searchable.where(id: account_ids)
+    def get(account, limit)
+      account_ids = redis.zrevrange("interactions:#{account.id}", 0, limit)
+
+      return [] if account_ids.empty? || limit < 1
+
+      accounts = Account.searchable.where(id: account_ids).index_by(&:id)
+
+      account_ids.map { |id| accounts[id.to_i] }.compact
     end
   end
 end
diff --git a/app/models/account.rb b/app/models/account.rb
index d85fd1f6e..80689d4aa 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -110,6 +110,7 @@ class Account < ApplicationRecord
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) }
   scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
+  scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
   scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
@@ -363,7 +364,7 @@ class Account < ApplicationRecord
   end
 
   def excluded_from_timeline_account_ids
-    Rails.cache.fetch("exclude_account_ids_for:#{id}") { blocking.pluck(:target_account_id) + blocked_by.pluck(:account_id) + muting.pluck(:target_account_id) }
+    Rails.cache.fetch("exclude_account_ids_for:#{id}") { block_relationships.pluck(:target_account_id) + blocked_by_relationships.pluck(:account_id) + mute_relationships.pluck(:target_account_id) }
   end
 
   def excluded_from_timeline_domains
diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb
new file mode 100644
index 000000000..7fe9d618e
--- /dev/null
+++ b/app/models/account_suggestions.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AccountSuggestions
+  class Suggestion < ActiveModelSerializers::Model
+    attributes :account, :source
+  end
+
+  def self.get(account, limit)
+    suggestions = PotentialFriendshipTracker.get(account, limit).map { |target_account| Suggestion.new(account: target_account, source: :past_interaction) }
+    suggestions.concat(FollowRecommendation.get(account, limit - suggestions.size, suggestions.map { |suggestion| suggestion.account.id }).map { |target_account| Suggestion.new(account: target_account, source: :global) }) if suggestions.size < limit
+    suggestions
+  end
+
+  def self.remove(account, target_account_id)
+    PotentialFriendshipTracker.remove(account.id, target_account_id)
+  end
+end
diff --git a/app/models/account_summary.rb b/app/models/account_summary.rb
new file mode 100644
index 000000000..6a7e17c6c
--- /dev/null
+++ b/app/models/account_summary.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: account_summaries
+#
+#  account_id :bigint(8)        primary key
+#  language   :string
+#  sensitive  :boolean
+#
+
+class AccountSummary < ApplicationRecord
+  self.primary_key = :account_id
+
+  scope :safe, -> { where(sensitive: false) }
+  scope :localized, ->(locale) { where(language: locale) }
+  scope :filtered, -> { joins(arel_table.join(FollowRecommendationSuppression.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:account_id].eq(FollowRecommendationSuppression.arel_table[:account_id])).join_sources).where(FollowRecommendationSuppression.arel_table[:id].eq(nil)) }
+
+  def self.refresh
+    Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false)
+  end
+
+  def readonly?
+    true
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 98849f8fc..aaf371ebd 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -63,5 +63,8 @@ module AccountAssociations
 
     # Account deletion requests
     has_one :deletion_request, class_name: 'AccountDeletionRequest', inverse_of: :account, dependent: :destroy
+
+    # Follow recommendations
+    has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy
   end
 end
diff --git a/app/models/follow_recommendation.rb b/app/models/follow_recommendation.rb
new file mode 100644
index 000000000..c4355224d
--- /dev/null
+++ b/app/models/follow_recommendation.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: follow_recommendations
+#
+#  account_id :bigint(8)        primary key
+#  rank       :decimal(, )
+#  reason     :text             is an Array
+#
+
+class FollowRecommendation < ApplicationRecord
+  self.primary_key = :account_id
+
+  belongs_to :account_summary, foreign_key: :account_id
+  belongs_to :account, foreign_key: :account_id
+
+  scope :safe, -> { joins(:account_summary).merge(AccountSummary.safe) }
+  scope :localized, ->(locale) { joins(:account_summary).merge(AccountSummary.localized(locale)) }
+  scope :filtered, -> { joins(:account_summary).merge(AccountSummary.filtered) }
+
+  def readonly?
+    true
+  end
+
+  def self.get(account, limit, exclude_account_ids = [])
+    account_ids = Redis.current.zrevrange("follow_recommendations:#{account.user_locale}", 0, -1).map(&:to_i) - exclude_account_ids - [account.id]
+
+    return [] if account_ids.empty? || limit < 1
+
+    accounts = Account.followable_by(account)
+                      .not_excluded_by_account(account)
+                      .not_domain_blocked_by_account(account)
+                      .where(id: account_ids)
+                      .limit(limit)
+                      .index_by(&:id)
+
+    account_ids.map { |id| accounts[id] }.compact
+  end
+end
diff --git a/app/models/follow_recommendation_filter.rb b/app/models/follow_recommendation_filter.rb
new file mode 100644
index 000000000..acf03cd84
--- /dev/null
+++ b/app/models/follow_recommendation_filter.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class FollowRecommendationFilter
+  KEYS = %i(
+    language
+    status
+  ).freeze
+
+  attr_reader :params, :language
+
+  def initialize(params)
+    @language = params.delete('language') || I18n.locale
+    @params   = params
+  end
+
+  def results
+    if params['status'] == 'suppressed'
+      Account.joins(:follow_recommendation_suppression).order(FollowRecommendationSuppression.arel_table[:id].desc).to_a
+    else
+      account_ids = Redis.current.zrevrange("follow_recommendations:#{@language}", 0, -1).map(&:to_i)
+      accounts    = Account.where(id: account_ids).index_by(&:id)
+
+      account_ids.map { |id| accounts[id] }.compact
+    end
+  end
+end
diff --git a/app/models/follow_recommendation_suppression.rb b/app/models/follow_recommendation_suppression.rb
new file mode 100644
index 000000000..170506b85
--- /dev/null
+++ b/app/models/follow_recommendation_suppression.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: follow_recommendation_suppressions
+#
+#  id         :bigint(8)        not null, primary key
+#  account_id :bigint(8)        not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class FollowRecommendationSuppression < ApplicationRecord
+  include Redisable
+
+  belongs_to :account
+
+  after_commit :remove_follow_recommendations, on: :create
+
+  private
+
+  def remove_follow_recommendations
+    redis.pipelined do
+      I18n.available_locales.each do |locale|
+        redis.zrem("follow_recommendations:#{locale}", account_id)
+      end
+    end
+  end
+end
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index 26d6d3abf..698933c9f 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -21,6 +21,10 @@ class Form::AccountBatch
       approve!
     when 'reject'
       reject!
+    when 'suppress_follow_recommendation'
+      suppress_follow_recommendation!
+    when 'unsuppress_follow_recommendation'
+      unsuppress_follow_recommendation!
     end
   end
 
@@ -79,4 +83,18 @@ class Form::AccountBatch
     records.each { |account| authorize(account.user, :reject?) }
            .each { |account| DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false) }
   end
+
+  def suppress_follow_recommendation!
+    authorize(:follow_recommendation, :suppress?)
+
+    accounts.each do |account|
+      FollowRecommendationSuppression.create(account: account)
+    end
+  end
+
+  def unsuppress_follow_recommendation!
+    authorize(:follow_recommendation, :unsuppress?)
+
+    FollowRecommendationSuppression.where(account_id: account_ids).destroy_all
+  end
 end
diff --git a/app/policies/follow_recommendation_policy.rb b/app/policies/follow_recommendation_policy.rb
new file mode 100644
index 000000000..68cd0e547
--- /dev/null
+++ b/app/policies/follow_recommendation_policy.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class FollowRecommendationPolicy < ApplicationPolicy
+  def show?
+    staff?
+  end
+
+  def suppress?
+    staff?
+  end
+
+  def unsuppress?
+    staff?
+  end
+end
diff --git a/app/serializers/rest/suggestion_serializer.rb b/app/serializers/rest/suggestion_serializer.rb
new file mode 100644
index 000000000..3d697fd9f
--- /dev/null
+++ b/app/serializers/rest/suggestion_serializer.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class REST::SuggestionSerializer < ActiveModel::Serializer
+  attributes :source
+
+  has_one :account, serializer: REST::AccountSerializer
+end
diff --git a/app/views/admin/follow_recommendations/_account.html.haml b/app/views/admin/follow_recommendations/_account.html.haml
new file mode 100644
index 000000000..af5a4aaf7
--- /dev/null
+++ b/app/views/admin/follow_recommendations/_account.html.haml
@@ -0,0 +1,20 @@
+.batch-table__row
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :account_ids, { multiple: true, include_hidden: false }, account.id
+  .batch-table__row__content.batch-table__row__content--unpadded
+    %table.accounts-table
+      %tbody
+        %tr
+          %td= account_link_to account
+          %td.accounts-table__count.optional
+            = number_to_human account.statuses_count, strip_insignificant_zeros: true
+            %small= t('accounts.posts', count: account.statuses_count).downcase
+          %td.accounts-table__count.optional
+            = number_to_human account.followers_count, strip_insignificant_zeros: true
+            %small= t('accounts.followers', count: account.followers_count).downcase
+          %td.accounts-table__count
+            - if account.last_status_at.present?
+              %time.time-ago{ datetime: account.last_status_at.to_date.iso8601, title: l(account.last_status_at.to_date) }= l account.last_status_at
+            - else
+              \-
+            %small= t('accounts.last_active')
diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml
new file mode 100644
index 000000000..1f050329a
--- /dev/null
+++ b/app/views/admin/follow_recommendations/show.html.haml
@@ -0,0 +1,42 @@
+- content_for :page_title do
+  = t('admin.follow_recommendations.title')
+
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
+
+.simple_form
+  %p.hint= t('admin.follow_recommendations.description_html')
+
+%hr.spacer/
+
+= form_tag admin_follow_recommendations_path, method: 'GET', class: 'simple_form' do
+  .filters
+    .filter-subset.filter-subset--with-select
+      %strong= t('admin.follow_recommendations.language')
+      .input.select.optional
+        = select_tag :language, options_for_select(I18n.available_locales.map { |key| [human_locale(key), key]}, @language)
+
+    .filter-subset
+      %strong= t('admin.follow_recommendations.status')
+      %ul
+        %li= filter_link_to t('admin.accounts.moderation.active'), status: nil
+        %li= filter_link_to t('admin.follow_recommendations.suppressed'), status: 'suppressed'
+
+= form_for(@form, url: admin_follow_recommendations_path, method: :patch) do |f|
+  - RelationshipFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        - if params[:status].blank? && can?(:suppress, :follow_recommendation)
+          = f.button safe_join([fa_icon('times'), t('admin.follow_recommendations.suppress')]), name: :suppress, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        - if params[:status] == 'suppressed' && can?(:unsuppress, :follow_recommendation)
+          = f.button safe_join([fa_icon('plus'), t('admin.follow_recommendations.unsuppress')]), name: :unsuppress, class: 'table-action-link', type: :submit
+    .batch-table__body
+      - if @accounts.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'account', collection: @accounts, locals: { f: f }
diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb
new file mode 100644
index 000000000..0a0286496
--- /dev/null
+++ b/app/workers/scheduler/follow_recommendations_scheduler.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class Scheduler::FollowRecommendationsScheduler
+  include Sidekiq::Worker
+  include Redisable
+
+  sidekiq_options retry: 0
+
+  # The maximum number of accounts that can be requested in one page from the
+  # API is 80, and the suggestions API does not allow pagination. This number
+  # leaves some room for accounts being filtered during live access
+  SET_SIZE = 100
+
+  def perform
+    # Maintaining a materialized view speeds-up subsequent queries significantly
+    AccountSummary.refresh
+
+    fallback_recommendations = FollowRecommendation.safe.filtered.limit(SET_SIZE).index_by(&:account_id)
+
+    I18n.available_locales.each do |locale|
+      recommendations = begin
+        if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist
+          FollowRecommendation.safe.filtered.localized(locale).limit(SET_SIZE).index_by(&:account_id)
+        else
+          {}
+        end
+      end
+
+      # Use language-agnostic results if there are not enough language-specific ones
+      missing = SET_SIZE - recommendations.keys.size
+
+      if missing.positive?
+        added = 0
+
+        # Avoid duplicate results
+        fallback_recommendations.each_value do |recommendation|
+          next if recommendations.key?(recommendation.account_id)
+
+          recommendations[recommendation.account_id] = recommendation
+          added += 1
+
+          break if added >= missing
+        end
+      end
+
+      redis.pipelined do
+        redis.del(key(locale))
+
+        recommendations.each_value do |recommendation|
+          redis.zadd(key(locale), recommendation.rank, recommendation.account_id)
+        end
+      end
+    end
+  end
+
+  private
+
+  def key(locale)
+    "follow_recommendations:#{locale}"
+  end
+end