Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `README.md`:
  Upstream updated copyright year, we don't mention it so kept our version.
- `app/controllers/admin/dashboard_controller.rb`:
  Not really a conflict, upstream change (removing the spam checker) too close
  to glitch-soc changes. Ported upstream changes.
- `app/models/form/admin_settings.rb`:
  Same.
- `app/services/remove_status_service.rb`:
  Same.
- `app/views/admin/settings/edit.html.haml`:
  Same.
- `config/settings.yml`:
  Same.
- `config/environments/production.rb`:
  Not a real conflict, upstream added a default HTTP header, but we have
  extra headers in glitch-soc.
  Added the header.

11 files changed, 265 insertions, 49 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index 2e7d9f543..8f042c931 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -114,6 +114,7 @@ class Account < ApplicationRecord
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) }
   scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
+  scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
   scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
@@ -238,6 +239,7 @@ class Account < ApplicationRecord
     transaction do
       create_deletion_request!
       update!(suspended_at: date, suspension_origin: origin)
+      create_canonical_email_block!
     end
   end
 
@@ -245,6 +247,7 @@ class Account < ApplicationRecord
     transaction do
       deletion_request&.destroy!
       update!(suspended_at: nil, suspension_origin: nil)
+      destroy_canonical_email_block!
     end
   end
 
@@ -365,7 +368,7 @@ class Account < ApplicationRecord
   end
 
   def excluded_from_timeline_account_ids
-    Rails.cache.fetch("exclude_account_ids_for:#{id}") { blocking.pluck(:target_account_id) + blocked_by.pluck(:account_id) + muting.pluck(:target_account_id) }
+    Rails.cache.fetch("exclude_account_ids_for:#{id}") { block_relationships.pluck(:target_account_id) + blocked_by_relationships.pluck(:account_id) + mute_relationships.pluck(:target_account_id) }
   end
 
   def excluded_from_timeline_domains
@@ -570,4 +573,16 @@ class Account < ApplicationRecord
   def clean_feed_manager
     FeedManager.instance.clean_feeds!(:home, [id])
   end
+
+  def create_canonical_email_block!
+    return unless local? && user_email.present?
+
+    CanonicalEmailBlock.create(reference_account: self, email: user_email)
+  end
+
+  def destroy_canonical_email_block!
+    return unless local?
+
+    CanonicalEmailBlock.where(reference_account: self).delete_all
+  end
 end
diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb
new file mode 100644
index 000000000..7fe9d618e
--- /dev/null
+++ b/app/models/account_suggestions.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AccountSuggestions
+  class Suggestion < ActiveModelSerializers::Model
+    attributes :account, :source
+  end
+
+  def self.get(account, limit)
+    suggestions = PotentialFriendshipTracker.get(account, limit).map { |target_account| Suggestion.new(account: target_account, source: :past_interaction) }
+    suggestions.concat(FollowRecommendation.get(account, limit - suggestions.size, suggestions.map { |suggestion| suggestion.account.id }).map { |target_account| Suggestion.new(account: target_account, source: :global) }) if suggestions.size < limit
+    suggestions
+  end
+
+  def self.remove(account, target_account_id)
+    PotentialFriendshipTracker.remove(account.id, target_account_id)
+  end
+end
diff --git a/app/models/account_summary.rb b/app/models/account_summary.rb
new file mode 100644
index 000000000..6a7e17c6c
--- /dev/null
+++ b/app/models/account_summary.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: account_summaries
+#
+#  account_id :bigint(8)        primary key
+#  language   :string
+#  sensitive  :boolean
+#
+
+class AccountSummary < ApplicationRecord
+  self.primary_key = :account_id
+
+  scope :safe, -> { where(sensitive: false) }
+  scope :localized, ->(locale) { where(language: locale) }
+  scope :filtered, -> { joins(arel_table.join(FollowRecommendationSuppression.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:account_id].eq(FollowRecommendationSuppression.arel_table[:account_id])).join_sources).where(FollowRecommendationSuppression.arel_table[:id].eq(nil)) }
+
+  def self.refresh
+    Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false)
+  end
+
+  def readonly?
+    true
+  end
+end
diff --git a/app/models/canonical_email_block.rb b/app/models/canonical_email_block.rb
new file mode 100644
index 000000000..a8546d65a
--- /dev/null
+++ b/app/models/canonical_email_block.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: canonical_email_blocks
+#
+#  id                   :bigint(8)        not null, primary key
+#  canonical_email_hash :string           default(""), not null
+#  reference_account_id :bigint(8)        not null
+#  created_at           :datetime         not null
+#  updated_at           :datetime         not null
+#
+
+class CanonicalEmailBlock < ApplicationRecord
+  include EmailHelper
+
+  belongs_to :reference_account, class_name: 'Account'
+
+  validates :canonical_email_hash, presence: true
+
+  def email=(email)
+    self.canonical_email_hash = email_to_canonical_email_hash(email)
+  end
+
+  def self.block?(email)
+    where(canonical_email_hash: email_to_canonical_email_hash(email)).exists?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 98849f8fc..aaf371ebd 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -63,5 +63,8 @@ module AccountAssociations
 
     # Account deletion requests
     has_one :deletion_request, class_name: 'AccountDeletionRequest', inverse_of: :account, dependent: :destroy
+
+    # Follow recommendations
+    has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy
   end
 end
diff --git a/app/models/follow_recommendation.rb b/app/models/follow_recommendation.rb
new file mode 100644
index 000000000..c4355224d
--- /dev/null
+++ b/app/models/follow_recommendation.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: follow_recommendations
+#
+#  account_id :bigint(8)        primary key
+#  rank       :decimal(, )
+#  reason     :text             is an Array
+#
+
+class FollowRecommendation < ApplicationRecord
+  self.primary_key = :account_id
+
+  belongs_to :account_summary, foreign_key: :account_id
+  belongs_to :account, foreign_key: :account_id
+
+  scope :safe, -> { joins(:account_summary).merge(AccountSummary.safe) }
+  scope :localized, ->(locale) { joins(:account_summary).merge(AccountSummary.localized(locale)) }
+  scope :filtered, -> { joins(:account_summary).merge(AccountSummary.filtered) }
+
+  def readonly?
+    true
+  end
+
+  def self.get(account, limit, exclude_account_ids = [])
+    account_ids = Redis.current.zrevrange("follow_recommendations:#{account.user_locale}", 0, -1).map(&:to_i) - exclude_account_ids - [account.id]
+
+    return [] if account_ids.empty? || limit < 1
+
+    accounts = Account.followable_by(account)
+                      .not_excluded_by_account(account)
+                      .not_domain_blocked_by_account(account)
+                      .where(id: account_ids)
+                      .limit(limit)
+                      .index_by(&:id)
+
+    account_ids.map { |id| accounts[id] }.compact
+  end
+end
diff --git a/app/models/follow_recommendation_filter.rb b/app/models/follow_recommendation_filter.rb
new file mode 100644
index 000000000..acf03cd84
--- /dev/null
+++ b/app/models/follow_recommendation_filter.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class FollowRecommendationFilter
+  KEYS = %i(
+    language
+    status
+  ).freeze
+
+  attr_reader :params, :language
+
+  def initialize(params)
+    @language = params.delete('language') || I18n.locale
+    @params   = params
+  end
+
+  def results
+    if params['status'] == 'suppressed'
+      Account.joins(:follow_recommendation_suppression).order(FollowRecommendationSuppression.arel_table[:id].desc).to_a
+    else
+      account_ids = Redis.current.zrevrange("follow_recommendations:#{@language}", 0, -1).map(&:to_i)
+      accounts    = Account.where(id: account_ids).index_by(&:id)
+
+      account_ids.map { |id| accounts[id] }.compact
+    end
+  end
+end
diff --git a/app/models/follow_recommendation_suppression.rb b/app/models/follow_recommendation_suppression.rb
new file mode 100644
index 000000000..170506b85
--- /dev/null
+++ b/app/models/follow_recommendation_suppression.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: follow_recommendation_suppressions
+#
+#  id         :bigint(8)        not null, primary key
+#  account_id :bigint(8)        not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class FollowRecommendationSuppression < ApplicationRecord
+  include Redisable
+
+  belongs_to :account
+
+  after_commit :remove_follow_recommendations, on: :create
+
+  private
+
+  def remove_follow_recommendations
+    redis.pipelined do
+      I18n.available_locales.each do |locale|
+        redis.zrem("follow_recommendations:#{locale}", account_id)
+      end
+    end
+  end
+end
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index 26d6d3abf..698933c9f 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -21,6 +21,10 @@ class Form::AccountBatch
       approve!
     when 'reject'
       reject!
+    when 'suppress_follow_recommendation'
+      suppress_follow_recommendation!
+    when 'unsuppress_follow_recommendation'
+      unsuppress_follow_recommendation!
     end
   end
 
@@ -79,4 +83,18 @@ class Form::AccountBatch
     records.each { |account| authorize(account.user, :reject?) }
            .each { |account| DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false) }
   end
+
+  def suppress_follow_recommendation!
+    authorize(:follow_recommendation, :suppress?)
+
+    accounts.each do |account|
+      FollowRecommendationSuppression.create(account: account)
+    end
+  end
+
+  def unsuppress_follow_recommendation!
+    authorize(:follow_recommendation, :unsuppress?)
+
+    FollowRecommendationSuppression.where(account_id: account_ids).destroy_all
+  end
 end
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 999d835e6..558a906d2 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -35,7 +35,6 @@ class Form::AdminSettings
     mascot
     show_reblogs_in_public_timelines
     show_replies_in_public_timelines
-    spam_check_enabled
     trends
     trendable_by_default
     show_domain_blocks
@@ -59,7 +58,6 @@ class Form::AdminSettings
     enable_keybase
     show_reblogs_in_public_timelines
     show_replies_in_public_timelines
-    spam_check_enabled
     trends
     trendable_by_default
     noindex
diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb
index c407a7789..6e46573ae 100644
--- a/app/models/web/push_subscription.rb
+++ b/app/models/web/push_subscription.rb
@@ -24,81 +24,101 @@ class Web::PushSubscription < ApplicationRecord
   validates :key_p256dh, presence: true
   validates :key_auth, presence: true
 
-  def push(notification)
-    I18n.with_locale(associated_user&.locale || I18n.default_locale) do
-      push_payload(payload_for_notification(notification), 48.hours.seconds)
-    end
+  delegate :locale, to: :associated_user
+
+  def encrypt(payload)
+    Webpush::Encryption.encrypt(payload, key_p256dh, key_auth)
+  end
+
+  def audience
+    @audience ||= Addressable::URI.parse(endpoint).normalized_site
+  end
+
+  def crypto_key_header
+    p256ecdsa = vapid_key.public_key_for_push_header
+
+    "p256ecdsa=#{p256ecdsa}"
+  end
+
+  def authorization_header
+    jwt = JWT.encode({ aud: audience, exp: 24.hours.from_now.to_i, sub: "mailto:#{contact_email}" }, vapid_key.curve, 'ES256', typ: 'JWT')
+
+    "WebPush #{jwt}"
   end
 
   def pushable?(notification)
-    data&.key?('alerts') && ActiveModel::Type::Boolean.new.cast(data['alerts'][notification.type.to_s])
+    policy_allows_notification?(notification) && alert_enabled_for_notification_type?(notification)
   end
 
   def associated_user
     return @associated_user if defined?(@associated_user)
 
-    @associated_user = if user_id.nil?
-                         session_activation.user
-                       else
-                         user
-                       end
+    @associated_user = begin
+      if user_id.nil?
+        session_activation.user
+      else
+        user
+      end
+    end
   end
 
   def associated_access_token
     return @associated_access_token if defined?(@associated_access_token)
 
-    @associated_access_token = if access_token_id.nil?
-                                 find_or_create_access_token.token
-                               else
-                                 access_token.token
-                               end
+    @associated_access_token = begin
+      if access_token_id.nil?
+        find_or_create_access_token.token
+      else
+        access_token.token
+      end
+    end
   end
 
   class << self
     def unsubscribe_for(application_id, resource_owner)
-      access_token_ids = Doorkeeper::AccessToken.where(application_id: application_id, resource_owner_id: resource_owner.id, revoked_at: nil)
-                                                .pluck(:id)
-
+      access_token_ids = Doorkeeper::AccessToken.where(application_id: application_id, resource_owner_id: resource_owner.id, revoked_at: nil).pluck(:id)
       where(access_token_id: access_token_ids).delete_all
     end
   end
 
   private
 
-  def push_payload(message, ttl = 5.minutes.seconds)
-    Webpush.payload_send(
-      message: Oj.dump(message),
-      endpoint: endpoint,
-      p256dh: key_p256dh,
-      auth: key_auth,
-      ttl: ttl,
-      ssl_timeout: 10,
-      open_timeout: 10,
-      read_timeout: 10,
-      vapid: {
-        subject: "mailto:#{::Setting.site_contact_email}",
-        private_key: Rails.configuration.x.vapid_private_key,
-        public_key: Rails.configuration.x.vapid_public_key,
-      }
-    )
-  end
-
-  def payload_for_notification(notification)
-    ActiveModelSerializers::SerializableResource.new(
-      notification,
-      serializer: Web::NotificationSerializer,
-      scope: self,
-      scope_name: :current_push_subscription
-    ).as_json
-  end
-
   def find_or_create_access_token
     Doorkeeper::AccessToken.find_or_create_for(
       application: Doorkeeper::Application.find_by(superapp: true),
-      resource_owner: session_activation.user_id,
+      resource_owner: user_id || session_activation.user_id,
       scopes: Doorkeeper::OAuth::Scopes.from_string('read write follow push'),
       expires_in: Doorkeeper.configuration.access_token_expires_in,
       use_refresh_token: Doorkeeper.configuration.refresh_token_enabled?
     )
   end
+
+  def vapid_key
+    @vapid_key ||= Webpush::VapidKey.from_keys(Rails.configuration.x.vapid_public_key, Rails.configuration.x.vapid_private_key)
+  end
+
+  def contact_email
+    @contact_email ||= ::Setting.site_contact_email
+  end
+
+  def alert_enabled_for_notification_type?(notification)
+    truthy?(data&.dig('alerts', notification.type.to_s))
+  end
+
+  def policy_allows_notification?(notification)
+    case data&.dig('policy')
+    when nil, 'all'
+      true
+    when 'none'
+      false
+    when 'followed'
+      notification.account.following?(notification.from_account)
+    when 'follower'
+      notification.from_account.following?(notification.account)
+    end
+  end
+
+  def truthy?(val)
+    ActiveModel::Type::Boolean.new.cast(val)
+  end
 end