From 50948b46aabc0756d85bc6641f0bd3bcc09bf7d4 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 20 Sep 2022 23:51:21 +0200
Subject: Add ability to filter followed accounts' posts by language (#19095)

---
 spec/models/concerns/account_interactions_spec.rb | 2 +-
 spec/models/export_spec.rb                        | 4 ++--
 spec/models/follow_request_spec.rb                | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'spec/models')

diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb
index 0369aff10..1d1898ab0 100644
--- a/spec/models/concerns/account_interactions_spec.rb
+++ b/spec/models/concerns/account_interactions_spec.rb
@@ -14,7 +14,7 @@ describe AccountInteractions do
     context 'account with Follow' do
       it 'returns { target_account_id => true }' do
         Fabricate(:follow, account: account, target_account: target_account)
-        is_expected.to eq(target_account_id => { reblogs: true, notify: false })
+        is_expected.to eq(target_account_id => { reblogs: true, notify: false, languages: nil })
       end
     end
 
diff --git a/spec/models/export_spec.rb b/spec/models/export_spec.rb
index 4e6b824bb..135d7a36b 100644
--- a/spec/models/export_spec.rb
+++ b/spec/models/export_spec.rb
@@ -35,8 +35,8 @@ describe Export do
       results = export.strip.split("\n")
 
       expect(results.size).to eq 3
-      expect(results.first).to eq 'Account address,Show boosts'
-      expect(results.second).to eq 'one@local.host,true'
+      expect(results.first).to eq 'Account address,Show boosts,Notify on new posts,Languages'
+      expect(results.second).to eq 'one@local.host,true,false,'
     end
   end
 
diff --git a/spec/models/follow_request_spec.rb b/spec/models/follow_request_spec.rb
index 36ce8ee60..901eabc9d 100644
--- a/spec/models/follow_request_spec.rb
+++ b/spec/models/follow_request_spec.rb
@@ -7,7 +7,7 @@ RSpec.describe FollowRequest, type: :model do
     let(:target_account) { Fabricate(:account) }
 
     it 'calls Account#follow!, MergeWorker.perform_async, and #destroy!' do
-      expect(account).to        receive(:follow!).with(target_account, reblogs: true, notify: false, uri: follow_request.uri, bypass_limit: true)
+      expect(account).to        receive(:follow!).with(target_account, reblogs: true, notify: false, uri: follow_request.uri, languages: nil, bypass_limit: true)
       expect(MergeWorker).to    receive(:perform_async).with(target_account.id, account.id)
       expect(follow_request).to receive(:destroy!)
       follow_request.authorize!
-- 
cgit 


From 9f65909f42c14d1e56c5f916eb76b156709ac147 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 5 Oct 2022 03:48:06 +0200
Subject: Change public timelines to be filtered by current locale by default
 (#19291)

In the absence of an opt-in to multiple specific languages in the
preferences, it makes more sense to filter by the user's presumed
language only (interface language or `lang` override)
---
 app/controllers/api/v1/timelines/public_controller.rb |  1 +
 app/controllers/api/v1/timelines/tag_controller.rb    |  1 +
 app/models/public_feed.rb                             | 13 ++++++++++++-
 app/models/status.rb                                  |  1 -
 app/models/tag_feed.rb                                |  2 ++
 spec/models/status_spec.rb                            | 16 ----------------
 6 files changed, 16 insertions(+), 18 deletions(-)

(limited to 'spec/models')

diff --git a/app/controllers/api/v1/timelines/public_controller.rb b/app/controllers/api/v1/timelines/public_controller.rb
index d253b744f..15b91d63e 100644
--- a/app/controllers/api/v1/timelines/public_controller.rb
+++ b/app/controllers/api/v1/timelines/public_controller.rb
@@ -35,6 +35,7 @@ class Api::V1::Timelines::PublicController < Api::BaseController
   def public_feed
     PublicFeed.new(
       current_account,
+      locale: content_locale,
       local: truthy_param?(:local),
       remote: truthy_param?(:remote),
       only_media: truthy_param?(:only_media)
diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb
index 64a1db58d..9f3a5b3f1 100644
--- a/app/controllers/api/v1/timelines/tag_controller.rb
+++ b/app/controllers/api/v1/timelines/tag_controller.rb
@@ -36,6 +36,7 @@ class Api::V1::Timelines::TagController < Api::BaseController
     TagFeed.new(
       @tag,
       current_account,
+      locale: content_locale,
       any: params[:any],
       all: params[:all],
       none: params[:none],
diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb
index 5e4c3e1ce..2cf9206d2 100644
--- a/app/models/public_feed.rb
+++ b/app/models/public_feed.rb
@@ -8,6 +8,7 @@ class PublicFeed
   # @option [Boolean] :local
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
+  # @option [String]  :locale
   def initialize(account, options = {})
     @account = account
     @options = options
@@ -27,6 +28,7 @@ class PublicFeed
     scope.merge!(remote_only_scope) if remote_only?
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
+    scope.merge!(language_scope)
 
     scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
@@ -83,10 +85,19 @@ class PublicFeed
     Status.joins(:media_attachments).group(:id)
   end
 
+  def language_scope
+    if account&.chosen_languages.present?
+      Status.where(language: account.chosen_languages)
+    elsif @options[:locale].present?
+      Status.where(language: @options[:locale])
+    else
+      Status.all
+    end
+  end
+
   def account_filters_scope
     Status.not_excluded_by_account(account).tap do |scope|
       scope.merge!(Status.not_domain_blocked_by_account(account)) unless local_only?
-      scope.merge!(Status.in_chosen_languages(account)) if account.chosen_languages.present?
     end
   end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 7eff990aa..de958aaf3 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -95,7 +95,6 @@ class Status < ApplicationRecord
   scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') }
   scope :with_public_visibility, -> { where(visibility: :public) }
   scope :tagged_with, ->(tag_ids) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag_ids }) }
-  scope :in_chosen_languages, ->(account) { where(language: nil).or where(language: account.chosen_languages) }
   scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced_at: nil }) }
   scope :including_silenced_accounts, -> { left_outer_joins(:account).where.not(accounts: { silenced_at: nil }) }
   scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) }
diff --git a/app/models/tag_feed.rb b/app/models/tag_feed.rb
index b8cd63557..58216bddc 100644
--- a/app/models/tag_feed.rb
+++ b/app/models/tag_feed.rb
@@ -12,6 +12,7 @@ class TagFeed < PublicFeed
   # @option [Boolean] :local
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
+  # @option [String]  :locale
   def initialize(tag, account, options = {})
     @tag = tag
     super(account, options)
@@ -32,6 +33,7 @@ class TagFeed < PublicFeed
     scope.merge!(remote_only_scope) if remote_only?
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
+    scope.merge!(language_scope)
 
     scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index 130f4d03f..78cc05959 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -251,22 +251,6 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe '.in_chosen_languages' do
-    context 'for accounts with language filters' do
-      let(:user) { Fabricate(:user, chosen_languages: ['en']) }
-
-      it 'does not include statuses in not in chosen languages' do
-        status = Fabricate(:status, language: 'de')
-        expect(Status.in_chosen_languages(user.account)).not_to include status
-      end
-
-      it 'includes status with unknown language' do
-        status = Fabricate(:status, language: nil)
-        expect(Status.in_chosen_languages(user.account)).to include status
-      end
-    end
-  end
-
   describe '.tagged_with' do
     let(:tag1)     { Fabricate(:tag) }
     let(:tag2)     { Fabricate(:tag) }
-- 
cgit 


From 45ebdb72ca1678eb30e6087f61bd019c7ea903f4 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 8 Oct 2022 16:45:40 +0200
Subject: Add support for language preferences for trending statuses and links
 (#18288)

---
 app/controllers/admin/trends/links_controller.rb   |  1 +
 .../admin/trends/statuses_controller.rb            |  1 +
 app/controllers/api/v1/trends/links_controller.rb  |  4 +-
 app/mailers/admin_mailer.rb                        |  4 +-
 app/models/preview_card.rb                         |  1 +
 app/models/preview_card_trend.rb                   | 17 ++++
 app/models/status.rb                               |  1 +
 app/models/status_trend.rb                         | 21 +++++
 app/models/trends.rb                               |  6 +-
 app/models/trends/base.rb                          |  4 +
 app/models/trends/links.rb                         | 98 +++++++++++++++-------
 app/models/trends/preview_card_filter.rb           | 25 ++++--
 app/models/trends/status_filter.rb                 | 25 ++++--
 app/models/trends/statuses.rb                      | 82 ++++++++++--------
 .../admin/trends/links/_preview_card.html.haml     |  4 +-
 app/views/admin/trends/links/index.html.haml       |  2 +-
 app/views/admin/trends/statuses/_status.html.haml  |  4 +-
 app/views/admin/trends/statuses/index.html.haml    |  2 +-
 .../admin_mailer/_new_trending_links.text.erb      |  8 +-
 .../admin_mailer/_new_trending_statuses.text.erb   |  8 +-
 config/locales/en.yml                              |  4 -
 db/migrate/20220824233535_create_status_trends.rb  | 12 +++
 .../20221006061337_create_preview_card_trends.rb   | 11 +++
 db/schema.rb                                       | 25 +++++-
 spec/fabricators/account_fabricator.rb             |  1 +
 spec/mailers/previews/admin_mailer_preview.rb      |  2 +-
 spec/models/preview_card_trend_spec.rb             |  4 +
 spec/models/status_trend_spec.rb                   |  4 +
 spec/models/trends/statuses_spec.rb                | 14 ++--
 29 files changed, 274 insertions(+), 121 deletions(-)
 create mode 100644 app/models/preview_card_trend.rb
 create mode 100644 app/models/status_trend.rb
 create mode 100644 db/migrate/20220824233535_create_status_trends.rb
 create mode 100644 db/migrate/20221006061337_create_preview_card_trends.rb
 create mode 100644 spec/models/preview_card_trend_spec.rb
 create mode 100644 spec/models/status_trend_spec.rb

(limited to 'spec/models')

diff --git a/app/controllers/admin/trends/links_controller.rb b/app/controllers/admin/trends/links_controller.rb
index a497eae41..a3454f2da 100644
--- a/app/controllers/admin/trends/links_controller.rb
+++ b/app/controllers/admin/trends/links_controller.rb
@@ -4,6 +4,7 @@ class Admin::Trends::LinksController < Admin::BaseController
   def index
     authorize :preview_card, :review?
 
+    @locales       = PreviewCardTrend.pluck('distinct language')
     @preview_cards = filtered_preview_cards.page(params[:page])
     @form          = Trends::PreviewCardBatch.new
   end
diff --git a/app/controllers/admin/trends/statuses_controller.rb b/app/controllers/admin/trends/statuses_controller.rb
index c538962f9..2b8226138 100644
--- a/app/controllers/admin/trends/statuses_controller.rb
+++ b/app/controllers/admin/trends/statuses_controller.rb
@@ -4,6 +4,7 @@ class Admin::Trends::StatusesController < Admin::BaseController
   def index
     authorize :status, :review?
 
+    @locales  = StatusTrend.pluck('distinct language')
     @statuses = filtered_statuses.page(params[:page])
     @form     = Trends::StatusBatch.new
   end
diff --git a/app/controllers/api/v1/trends/links_controller.rb b/app/controllers/api/v1/trends/links_controller.rb
index 1a9f918f2..8ff3b364e 100644
--- a/app/controllers/api/v1/trends/links_controller.rb
+++ b/app/controllers/api/v1/trends/links_controller.rb
@@ -28,7 +28,9 @@ class Api::V1::Trends::LinksController < Api::BaseController
   end
 
   def links_from_trends
-    Trends.links.query.allowed.in_locale(content_locale)
+    scope = Trends.links.query.allowed.in_locale(content_locale)
+    scope = scope.filtered_for(current_account) if user_signed_in?
+    scope
   end
 
   def insert_pagination_headers
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index f416977d8..bc6d87ae6 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -4,6 +4,7 @@ class AdminMailer < ApplicationMailer
   layout 'plain_mailer'
 
   helper :accounts
+  helper :languages
 
   def new_report(recipient, report)
     @report   = report
@@ -37,11 +38,8 @@ class AdminMailer < ApplicationMailer
 
   def new_trends(recipient, links, tags, statuses)
     @links                  = links
-    @lowest_trending_link   = Trends.links.query.allowed.limit(Trends.links.options[:review_threshold]).last
     @tags                   = tags
-    @lowest_trending_tag    = Trends.tags.query.allowed.limit(Trends.tags.options[:review_threshold]).last
     @statuses               = statuses
-    @lowest_trending_status = Trends.statuses.query.allowed.limit(Trends.statuses.options[:review_threshold]).last
     @me                     = recipient
     @instance               = Rails.configuration.x.local_domain
 
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index c49c51a1b..b5d3f9c8f 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -48,6 +48,7 @@ class PreviewCard < ApplicationRecord
   enum link_type: [:unknown, :article]
 
   has_and_belongs_to_many :statuses
+  has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy
 
   has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' }, validate_media_type: false
 
diff --git a/app/models/preview_card_trend.rb b/app/models/preview_card_trend.rb
new file mode 100644
index 000000000..018400dfa
--- /dev/null
+++ b/app/models/preview_card_trend.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: preview_card_trends
+#
+#  id              :bigint(8)        not null, primary key
+#  preview_card_id :bigint(8)        not null
+#  score           :float            default(0.0), not null
+#  rank            :integer          default(0), not null
+#  allowed         :boolean          default(FALSE), not null
+#  language        :string
+#
+class PreviewCardTrend < ApplicationRecord
+  belongs_to :preview_card
+  scope :allowed, -> { where(allowed: true) }
+end
diff --git a/app/models/status.rb b/app/models/status.rb
index de958aaf3..4805abfea 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -75,6 +75,7 @@ class Status < ApplicationRecord
   has_one :notification, as: :activity, dependent: :destroy
   has_one :status_stat, inverse_of: :status
   has_one :poll, inverse_of: :status, dependent: :destroy
+  has_one :trend, class_name: 'StatusTrend', inverse_of: :status
 
   validates :uri, uniqueness: true, presence: true, unless: :local?
   validates :text, presence: true, unless: -> { with_media? || reblog? }
diff --git a/app/models/status_trend.rb b/app/models/status_trend.rb
new file mode 100644
index 000000000..33fd6b004
--- /dev/null
+++ b/app/models/status_trend.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: status_trends
+#
+#  id         :bigint(8)        not null, primary key
+#  status_id  :bigint(8)        not null
+#  account_id :bigint(8)        not null
+#  score      :float            default(0.0), not null
+#  rank       :integer          default(0), not null
+#  allowed    :boolean          default(FALSE), not null
+#  language   :string
+#
+
+class StatusTrend < ApplicationRecord
+  belongs_to :status
+  belongs_to :account
+
+  scope :allowed, -> { where(allowed: true) }
+end
diff --git a/app/models/trends.rb b/app/models/trends.rb
index d886be89a..151d734f9 100644
--- a/app/models/trends.rb
+++ b/app/models/trends.rb
@@ -26,7 +26,7 @@ module Trends
   end
 
   def self.request_review!
-    return unless enabled?
+    return if skip_review? || !enabled?
 
     links_requiring_review    = links.request_review
     tags_requiring_review     = tags.request_review
@@ -43,6 +43,10 @@ module Trends
     Setting.trends
   end
 
+  def skip_review?
+    Setting.trendable_by_default
+  end
+
   def self.available_locales
     @available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq
   end
diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb
index 047111248..a189f11f2 100644
--- a/app/models/trends/base.rb
+++ b/app/models/trends/base.rb
@@ -98,4 +98,8 @@ class Trends::Base
       pipeline.rename(from_key, to_key)
     end
   end
+
+  def skip_review?
+    Setting.trendable_by_default
+  end
 end
diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb
index 604894cd6..8808b3ab6 100644
--- a/app/models/trends/links.rb
+++ b/app/models/trends/links.rb
@@ -11,6 +11,40 @@ class Trends::Links < Trends::Base
     decay_threshold: 1,
   }
 
+  class Query < Trends::Query
+    def filtered_for!(account)
+      @account = account
+      self
+    end
+
+    def filtered_for(account)
+      clone.filtered_for!(account)
+    end
+
+    def to_arel
+      scope = PreviewCard.joins(:trend).reorder(score: :desc)
+      scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
+      scope = scope.merge(PreviewCardTrend.allowed) if @allowed
+      scope = scope.offset(@offset) if @offset.present?
+      scope = scope.limit(@limit) if @limit.present?
+      scope
+    end
+
+    private
+
+    def language_order_clause
+      Arel::Nodes::Case.new.when(PreviewCardTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
+    end
+
+    def preferred_languages
+      if @account&.chosen_languages.present?
+        @account.chosen_languages
+      else
+        @locale
+      end
+    end
+  end
+
   def register(status, at_time = Time.now.utc)
     original_status = status.proper
 
@@ -28,24 +62,33 @@ class Trends::Links < Trends::Base
     record_used_id(preview_card.id, at_time)
   end
 
+  def query
+    Query.new(key_prefix, klass)
+  end
+
   def refresh(at_time = Time.now.utc)
-    preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + PreviewCardTrend.pluck(:preview_card_id)).uniq)
     calculate_scores(preview_cards, at_time)
   end
 
   def request_review
-    preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
+    PreviewCardTrend.pluck('distinct language').flat_map do |language|
+      score_at_threshold  = PreviewCardTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0
+      preview_card_trends = PreviewCardTrend.where(language: language, allowed: false).joins(:preview_card)
 
-    preview_cards.filter_map do |preview_card|
-      next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
+      preview_card_trends.filter_map do |trend|
+        preview_card = trend.preview_card
 
-      if preview_card.provider.nil?
-        preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
-      else
-        preview_card.provider.touch(:requested_review_at)
-      end
+        next unless trend.score > score_at_threshold && !preview_card.trendable? && preview_card.requires_review_notification?
+
+        if preview_card.provider.nil?
+          preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
+        else
+          preview_card.provider.touch(:requested_review_at)
+        end
 
-      preview_card
+        preview_card
+      end
     end
   end
 
@@ -62,10 +105,7 @@ class Trends::Links < Trends::Base
   private
 
   def calculate_scores(preview_cards, at_time)
-    global_items = []
-    locale_items = Hash.new { |h, key| h[key] = [] }
-
-    preview_cards.each do |preview_card|
+    items = preview_cards.map do |preview_card|
       expected  = preview_card.history.get(at_time - 1.day).accounts.to_f
       expected  = 1.0 if expected.zero?
       observed  = preview_card.history.get(at_time).accounts.to_f
@@ -89,26 +129,24 @@ class Trends::Links < Trends::Base
         preview_card.update_columns(max_score: max_score, max_score_at: max_time)
       end
 
-      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
-
-      next unless decaying_score >= options[:decay_threshold]
+      decaying_score = begin
+        if max_score.zero? || !valid_locale?(preview_card.language)
+          0
+        else
+          max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+        end
+      end
 
-      global_items << { score: decaying_score, item:  preview_card }
-      locale_items[preview_card.language] << { score: decaying_score, item: preview_card } if valid_locale?(preview_card.language)
+      [decaying_score, preview_card]
     end
 
-    replace_items('', global_items)
+    to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
+    to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
 
-    Trends.available_locales.each do |locale|
-      replace_items(":#{locale}", locale_items[locale])
+    PreviewCardTrend.transaction do
+      PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any?
+      PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any?
+      PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id')
     end
   end
-
-  def filter_for_allowed_items(items)
-    items.select { |item| item[:item].trendable? }
-  end
-
-  def would_be_trending?(id)
-    score(id) > score_at_rank(options[:review_threshold] - 1)
-  end
 end
diff --git a/app/models/trends/preview_card_filter.rb b/app/models/trends/preview_card_filter.rb
index 25add58c8..0a81146d4 100644
--- a/app/models/trends/preview_card_filter.rb
+++ b/app/models/trends/preview_card_filter.rb
@@ -13,10 +13,10 @@ class Trends::PreviewCardFilter
   end
 
   def results
-    scope = PreviewCard.unscoped
+    scope = initial_scope
 
     params.each do |key, value|
-      next if %w(page locale).include?(key.to_s)
+      next if %w(page).include?(key.to_s)
 
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end
@@ -26,21 +26,30 @@ class Trends::PreviewCardFilter
 
   private
 
+  def initial_scope
+    PreviewCard.select(PreviewCard.arel_table[Arel.star])
+               .joins(:trend)
+               .eager_load(:trend)
+               .reorder(score: :desc)
+  end
+
   def scope_for(key, value)
     case key.to_s
     when 'trending'
       trending_scope(value)
+    when 'locale'
+      PreviewCardTrend.where(language: value)
     else
       raise "Unknown filter: #{key}"
     end
   end
 
   def trending_scope(value)
-    scope = Trends.links.query
-
-    scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
-    scope = scope.allowed if value == 'allowed'
-
-    scope.to_arel
+    case value
+    when 'allowed'
+      PreviewCardTrend.allowed
+    else
+      PreviewCardTrend.all
+    end
   end
 end
diff --git a/app/models/trends/status_filter.rb b/app/models/trends/status_filter.rb
index 7c453e339..cb0f75d67 100644
--- a/app/models/trends/status_filter.rb
+++ b/app/models/trends/status_filter.rb
@@ -13,10 +13,10 @@ class Trends::StatusFilter
   end
 
   def results
-    scope = Status.unscoped.kept
+    scope = initial_scope
 
     params.each do |key, value|
-      next if %w(page locale).include?(key.to_s)
+      next if %w(page).include?(key.to_s)
 
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end
@@ -26,21 +26,30 @@ class Trends::StatusFilter
 
   private
 
+  def initial_scope
+    Status.select(Status.arel_table[Arel.star])
+          .joins(:trend)
+          .eager_load(:trend)
+          .reorder(score: :desc)
+  end
+
   def scope_for(key, value)
     case key.to_s
     when 'trending'
       trending_scope(value)
+    when 'locale'
+      StatusTrend.where(language: value)
     else
       raise "Unknown filter: #{key}"
     end
   end
 
   def trending_scope(value)
-    scope = Trends.statuses.query
-
-    scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
-    scope = scope.allowed if value == 'allowed'
-
-    scope.to_arel
+    case value
+    when 'allowed'
+      StatusTrend.allowed
+    else
+      StatusTrend.all
+    end
   end
 end
diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb
index 777065d3e..c9ef7c8f2 100644
--- a/app/models/trends/statuses.rb
+++ b/app/models/trends/statuses.rb
@@ -20,13 +20,27 @@ class Trends::Statuses < Trends::Base
       clone.filtered_for!(account)
     end
 
+    def to_arel
+      scope = Status.joins(:trend).reorder(score: :desc)
+      scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
+      scope = scope.merge(StatusTrend.allowed) if @allowed
+      scope = scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) if @account.present?
+      scope = scope.offset(@offset) if @offset.present?
+      scope = scope.limit(@limit) if @limit.present?
+      scope
+    end
+
     private
 
-    def apply_scopes(scope)
-      if @account.nil?
-        scope
+    def language_order_clause
+      Arel::Nodes::Case.new.when(StatusTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
+    end
+
+    def preferred_languages
+      if @account&.chosen_languages.present?
+        @account.chosen_languages
       else
-        scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account)
+        @locale
       end
     end
   end
@@ -36,9 +50,6 @@ class Trends::Statuses < Trends::Base
   end
 
   def add(status, _account_id, at_time = Time.now.utc)
-    # We rely on the total reblogs and favourites count, so we
-    # don't record which account did the what and when here
-
     record_used_id(status.id, at_time)
   end
 
@@ -47,18 +58,23 @@ class Trends::Statuses < Trends::Base
   end
 
   def refresh(at_time = Time.now.utc)
-    statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
+    statuses = Status.where(id: (recently_used_ids(at_time) + StatusTrend.pluck(:status_id)).uniq).includes(:status_stat, :account)
     calculate_scores(statuses, at_time)
   end
 
   def request_review
-    statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account)
+    StatusTrend.pluck('distinct language').flat_map do |language|
+      score_at_threshold = StatusTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0
+      status_trends      = StatusTrend.where(language: language, allowed: false).joins(:status).includes(status: :account)
 
-    statuses.filter_map do |status|
-      next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification?
+      status_trends.filter_map do |trend|
+        status = trend.status
 
-      status.account.touch(:requested_review_at)
-      status
+        if trend.score > score_at_threshold && !status.trendable? && status.requires_review_notification?
+          status.account.touch(:requested_review_at)
+          status
+        end
+      end
     end
   end
 
@@ -75,14 +91,11 @@ class Trends::Statuses < Trends::Base
   private
 
   def eligible?(status)
-    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply?
+    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
   end
 
   def calculate_scores(statuses, at_time)
-    global_items = []
-    locale_items = Hash.new { |h, key| h[key] = [] }
-
-    statuses.each do |status|
+    items = statuses.map do |status|
       expected  = 1.0
       observed  = (status.reblogs_count + status.favourites_count).to_f
 
@@ -94,29 +107,24 @@ class Trends::Statuses < Trends::Base
         end
       end
 
-      decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
-
-      next unless decaying_score >= options[:decay_threshold]
+      decaying_score = begin
+        if score.zero? || !eligible?(status)
+          0
+        else
+          score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
+        end
+      end
 
-      global_items << { score: decaying_score, item: status }
-      locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language)
+      [decaying_score, status]
     end
 
-    replace_items('', global_items)
+    to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
+    to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
 
-    Trends.available_locales.each do |locale|
-      replace_items(":#{locale}", locale_items[locale])
+    StatusTrend.transaction do
+      StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any?
+      StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any?
+      StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id')
     end
   end
-
-  def filter_for_allowed_items(items)
-    # Show only one status per account, pick the one with the highest score
-    # that's also eligible to trend
-
-    items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } }
-  end
-
-  def would_be_trending?(id)
-    score(id) > score_at_rank(options[:review_threshold] - 1)
-  end
 end
diff --git a/app/views/admin/trends/links/_preview_card.html.haml b/app/views/admin/trends/links/_preview_card.html.haml
index 7d4897c7e..8812feb31 100644
--- a/app/views/admin/trends/links/_preview_card.html.haml
+++ b/app/views/admin/trends/links/_preview_card.html.haml
@@ -18,9 +18,9 @@
 
       = t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts })
 
-      - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id, locale: params[:locale].presence))
+      - if preview_card.trend.allowed?
         •
-        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id, locale: params[:locale].presence)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+        %abbr{ title: t('admin.trends.tags.current_score', score: preview_card.trend.score) }= t('admin.trends.tags.trending_rank', rank: preview_card.trend.rank)
 
         - if preview_card.decaying?
           •
diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml
index 49a53d979..7b5122cf1 100644
--- a/app/views/admin/trends/links/index.html.haml
+++ b/app/views/admin/trends/links/index.html.haml
@@ -16,7 +16,7 @@
     .filter-subset.filter-subset--with-select
       %strong= t('admin.follow_recommendations.language')
       .input.select.optional
-        = select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, params[:locale]), include_blank: true
+        = select_tag :locale, options_for_select(@locales.map { |key| [standard_locale_name(key), key] }, params[:locale]), include_blank: true
     .filter-subset
       %strong= t('admin.trends.trending')
       %ul
diff --git a/app/views/admin/trends/statuses/_status.html.haml b/app/views/admin/trends/statuses/_status.html.haml
index e4d75bbb9..f35e13d12 100644
--- a/app/views/admin/trends/statuses/_status.html.haml
+++ b/app/views/admin/trends/statuses/_status.html.haml
@@ -25,9 +25,9 @@
     - if status.trendable? && !status.account.discoverable?
       •
       = t('admin.trends.statuses.not_discoverable')
-    - if status.trendable? && (rank = Trends.statuses.rank(status.id, locale: params[:locale].presence))
+    - if status.trend.allowed?
       •
-      %abbr{ title: t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id, locale: params[:locale].presence)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+      %abbr{ title: t('admin.trends.tags.current_score', score: status.trend.score) }= t('admin.trends.tags.trending_rank', rank: status.trend.rank)
     - elsif status.requires_review?
       •
       = t('admin.trends.pending_review')
diff --git a/app/views/admin/trends/statuses/index.html.haml b/app/views/admin/trends/statuses/index.html.haml
index b0059b20d..a42d60b00 100644
--- a/app/views/admin/trends/statuses/index.html.haml
+++ b/app/views/admin/trends/statuses/index.html.haml
@@ -16,7 +16,7 @@
     .filter-subset.filter-subset--with-select
       %strong= t('admin.follow_recommendations.language')
       .input.select.optional
-        = select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key]}, params[:locale]), include_blank: true
+        = select_tag :locale, options_for_select(@locales.map { |key| [standard_locale_name(key), key] }, params[:locale]), include_blank: true
     .filter-subset
       %strong= t('admin.trends.trending')
       %ul
diff --git a/app/views/admin_mailer/_new_trending_links.text.erb b/app/views/admin_mailer/_new_trending_links.text.erb
index 405926fdd..602e12793 100644
--- a/app/views/admin_mailer/_new_trending_links.text.erb
+++ b/app/views/admin_mailer/_new_trending_links.text.erb
@@ -2,13 +2,7 @@
 
 <% @links.each do |link| %>
 - <%= link.title %> • <%= link.url %>
-  <%= raw t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
-<% end %>
-
-<% if @lowest_trending_link %>
-<%= raw t('admin_mailer.new_trends.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2), rank: Trends.links.options[:review_threshold]) %>
-<% else %>
-<%= raw t('admin_mailer.new_trends.new_trending_links.no_approved_links') %>
+  <%= standard_locale_name(link.language) %> • <%= raw t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: link.trend.score.round(2)) %>
 <% end %>
 
 <%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>
diff --git a/app/views/admin_mailer/_new_trending_statuses.text.erb b/app/views/admin_mailer/_new_trending_statuses.text.erb
index 8d11a80c2..1ed3ae857 100644
--- a/app/views/admin_mailer/_new_trending_statuses.text.erb
+++ b/app/views/admin_mailer/_new_trending_statuses.text.erb
@@ -2,13 +2,7 @@
 
 <% @statuses.each do |status| %>
 - <%= ActivityPub::TagManager.instance.url_for(status) %>
-  <%= raw t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id).round(2)) %>
-<% end %>
-
-<% if @lowest_trending_status %>
-<%= raw t('admin_mailer.new_trends.new_trending_statuses.requirements', lowest_status_url: ActivityPub::TagManager.instance.url_for(@lowest_trending_status), lowest_status_score: Trends.statuses.score(@lowest_trending_status.id).round(2), rank: Trends.statuses.options[:review_threshold]) %>
-<% else %>
-<%= raw t('admin_mailer.new_trends.new_trending_statuses.no_approved_statuses') %>
+  <%= standard_locale_name(status.language) %> • <%= raw t('admin.trends.tags.current_score', score: status.trend.score.round(2)) %>
 <% end %>
 
 <%= raw t('application_mailer.view')%> <%= admin_trends_statuses_url %>
diff --git a/config/locales/en.yml b/config/locales/en.yml
index cdac4fb54..bd913a5ca 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -939,12 +939,8 @@ en:
     new_trends:
       body: 'The following items need a review before they can be displayed publicly:'
       new_trending_links:
-        no_approved_links: There are currently no approved trending links.
-        requirements: 'Any of these candidates could surpass the #%{rank} approved trending link, which is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.'
         title: Trending links
       new_trending_statuses:
-        no_approved_statuses: There are currently no approved trending posts.
-        requirements: 'Any of these candidates could surpass the #%{rank} approved trending post, which is currently %{lowest_status_url} with a score of %{lowest_status_score}.'
         title: Trending posts
       new_trending_tags:
         no_approved_tags: There are currently no approved trending hashtags.
diff --git a/db/migrate/20220824233535_create_status_trends.rb b/db/migrate/20220824233535_create_status_trends.rb
new file mode 100644
index 000000000..cea0abf35
--- /dev/null
+++ b/db/migrate/20220824233535_create_status_trends.rb
@@ -0,0 +1,12 @@
+class CreateStatusTrends < ActiveRecord::Migration[6.1]
+  def change
+    create_table :status_trends do |t|
+      t.references :status, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true }
+      t.references :account, null: false, foreign_key: { on_delete: :cascade }
+      t.float :score, null: false, default: 0
+      t.integer :rank, null: false, default: 0
+      t.boolean :allowed, null: false, default: false
+      t.string :language
+    end
+  end
+end
diff --git a/db/migrate/20221006061337_create_preview_card_trends.rb b/db/migrate/20221006061337_create_preview_card_trends.rb
new file mode 100644
index 000000000..baad9c31c
--- /dev/null
+++ b/db/migrate/20221006061337_create_preview_card_trends.rb
@@ -0,0 +1,11 @@
+class CreatePreviewCardTrends < ActiveRecord::Migration[6.1]
+  def change
+    create_table :preview_card_trends do |t|
+      t.references :preview_card, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true }
+      t.float :score, null: false, default: 0
+      t.integer :rank, null: false, default: 0
+      t.boolean :allowed, null: false, default: false
+      t.string :language
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1a98b22db..2f9058509 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2022_08_29_192658) do
+ActiveRecord::Schema.define(version: 2022_10_06_061337) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -735,6 +735,15 @@ ActiveRecord::Schema.define(version: 2022_08_29_192658) do
     t.index ["domain"], name: "index_preview_card_providers_on_domain", unique: true
   end
 
+  create_table "preview_card_trends", force: :cascade do |t|
+    t.bigint "preview_card_id", null: false
+    t.float "score", default: 0.0, null: false
+    t.integer "rank", default: 0, null: false
+    t.boolean "allowed", default: false, null: false
+    t.string "language"
+    t.index ["preview_card_id"], name: "index_preview_card_trends_on_preview_card_id", unique: true
+  end
+
   create_table "preview_cards", force: :cascade do |t|
     t.string "url", default: "", null: false
     t.string "title", default: "", null: false
@@ -894,6 +903,17 @@ ActiveRecord::Schema.define(version: 2022_08_29_192658) do
     t.index ["status_id"], name: "index_status_stats_on_status_id", unique: true
   end
 
+  create_table "status_trends", force: :cascade do |t|
+    t.bigint "status_id", null: false
+    t.bigint "account_id", null: false
+    t.float "score", default: 0.0, null: false
+    t.integer "rank", default: 0, null: false
+    t.boolean "allowed", default: false, null: false
+    t.string "language"
+    t.index ["account_id"], name: "index_status_trends_on_account_id"
+    t.index ["status_id"], name: "index_status_trends_on_status_id", unique: true
+  end
+
   create_table "statuses", id: :bigint, default: -> { "timestamp_id('statuses'::text)" }, force: :cascade do |t|
     t.string "uri"
     t.text "text", default: "", null: false
@@ -1171,6 +1191,7 @@ ActiveRecord::Schema.define(version: 2022_08_29_192658) do
   add_foreign_key "poll_votes", "polls", on_delete: :cascade
   add_foreign_key "polls", "accounts", on_delete: :cascade
   add_foreign_key "polls", "statuses", on_delete: :cascade
+  add_foreign_key "preview_card_trends", "preview_cards", on_delete: :cascade
   add_foreign_key "report_notes", "accounts", on_delete: :cascade
   add_foreign_key "report_notes", "reports", on_delete: :cascade
   add_foreign_key "reports", "accounts", column: "action_taken_by_account_id", name: "fk_bca45b75fd", on_delete: :nullify
@@ -1185,6 +1206,8 @@ ActiveRecord::Schema.define(version: 2022_08_29_192658) do
   add_foreign_key "status_pins", "accounts", name: "fk_d4cb435b62", on_delete: :cascade
   add_foreign_key "status_pins", "statuses", on_delete: :cascade
   add_foreign_key "status_stats", "statuses", on_delete: :cascade
+  add_foreign_key "status_trends", "accounts", on_delete: :cascade
+  add_foreign_key "status_trends", "statuses", on_delete: :cascade
   add_foreign_key "statuses", "accounts", column: "in_reply_to_account_id", name: "fk_c7fa917661", on_delete: :nullify
   add_foreign_key "statuses", "accounts", name: "fk_9bda1543f7", on_delete: :cascade
   add_foreign_key "statuses", "statuses", column: "in_reply_to_id", on_delete: :nullify
diff --git a/spec/fabricators/account_fabricator.rb b/spec/fabricators/account_fabricator.rb
index f1cce281c..205706532 100644
--- a/spec/fabricators/account_fabricator.rb
+++ b/spec/fabricators/account_fabricator.rb
@@ -11,4 +11,5 @@ Fabricator(:account) do
   suspended_at        { |attrs| attrs[:suspended] ? Time.now.utc : nil }
   silenced_at         { |attrs| attrs[:silenced] ? Time.now.utc : nil }
   user                { |attrs| attrs[:domain].nil? ? Fabricate.build(:user, account: nil) : nil }
+  discoverable        true
 end
diff --git a/spec/mailers/previews/admin_mailer_preview.rb b/spec/mailers/previews/admin_mailer_preview.rb
index 01436ba7a..0ec9e9882 100644
--- a/spec/mailers/previews/admin_mailer_preview.rb
+++ b/spec/mailers/previews/admin_mailer_preview.rb
@@ -8,7 +8,7 @@ class AdminMailerPreview < ActionMailer::Preview
 
   # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trends
   def new_trends
-    AdminMailer.new_trends(Account.first, PreviewCard.limit(3), Tag.limit(3), Status.where(reblog_of_id: nil).limit(3))
+    AdminMailer.new_trends(Account.first, PreviewCard.joins(:trend).limit(3), Tag.limit(3), Status.joins(:trend).where(reblog_of_id: nil).limit(3))
   end
 
   # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_appeal
diff --git a/spec/models/preview_card_trend_spec.rb b/spec/models/preview_card_trend_spec.rb
new file mode 100644
index 000000000..c7ab6ed14
--- /dev/null
+++ b/spec/models/preview_card_trend_spec.rb
@@ -0,0 +1,4 @@
+require 'rails_helper'
+
+RSpec.describe PreviewCardTrend, type: :model do
+end
diff --git a/spec/models/status_trend_spec.rb b/spec/models/status_trend_spec.rb
new file mode 100644
index 000000000..6b82204a6
--- /dev/null
+++ b/spec/models/status_trend_spec.rb
@@ -0,0 +1,4 @@
+require 'rails_helper'
+
+RSpec.describe StatusTrend, type: :model do
+end
diff --git a/spec/models/trends/statuses_spec.rb b/spec/models/trends/statuses_spec.rb
index 9cc67acbe..5f338a65e 100644
--- a/spec/models/trends/statuses_spec.rb
+++ b/spec/models/trends/statuses_spec.rb
@@ -9,8 +9,8 @@ RSpec.describe Trends::Statuses do
     let!(:query) { subject.query }
     let!(:today) { at_time }
 
-    let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: today) }
-    let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) }
+    let!(:status1) { Fabricate(:status, text: 'Foo', language: 'en', trendable: true, created_at: today) }
+    let!(:status2) { Fabricate(:status, text: 'Bar', language: 'en', trendable: true, created_at: today) }
 
     before do
       15.times { reblog(status1, today) }
@@ -69,9 +69,9 @@ RSpec.describe Trends::Statuses do
     let!(:today) { at_time }
     let!(:yesterday) { today - 1.day }
 
-    let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: yesterday) }
-    let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) }
-    let!(:status3) { Fabricate(:status, text: 'Baz', trendable: true, created_at: today) }
+    let!(:status1) { Fabricate(:status, text: 'Foo', language: 'en', trendable: true, created_at: yesterday) }
+    let!(:status2) { Fabricate(:status, text: 'Bar', language: 'en', trendable: true, created_at: today) }
+    let!(:status3) { Fabricate(:status, text: 'Baz', language: 'en', trendable: true, created_at: today) }
 
     before do
       13.times { reblog(status1, today) }
@@ -95,10 +95,10 @@ RSpec.describe Trends::Statuses do
 
     it 'decays scores' do
       subject.refresh(today)
-      original_score = subject.score(status2.id)
+      original_score = status2.trend.score
       expect(original_score).to be_a Float
       subject.refresh(today + subject.options[:score_halflife])
-      decayed_score = subject.score(status2.id)
+      decayed_score = status2.trend.reload.score
       expect(decayed_score).to be <= original_score / 2
     end
   end
-- 
cgit 


From d0ba77047e539b7ae102296d096fcfd668a2ec92 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 1 Nov 2022 13:01:39 +0100
Subject: Change max. thumbnail dimensions to 640x360px (360p) (#19619)

---
 app/models/media_attachment.rb       | 2 +-
 spec/models/media_attachment_spec.rb | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'spec/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index a24fb857b..4a1948380 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -72,7 +72,7 @@ class MediaAttachment < ApplicationRecord
     }.freeze,
 
     small: {
-      pixels: 160_000, # 400x400px
+      pixels: 230_400, # 640x360px
       file_geometry_parser: FastGeometryParser,
       blurhash: BLURHASH_OPTIONS,
     }.freeze,
diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb
index cbd9a09c5..29fd313ae 100644
--- a/spec/models/media_attachment_spec.rb
+++ b/spec/models/media_attachment_spec.rb
@@ -157,9 +157,9 @@ RSpec.describe MediaAttachment, type: :model do
       expect(media.file.meta["original"]["width"]).to eq 600
       expect(media.file.meta["original"]["height"]).to eq 400
       expect(media.file.meta["original"]["aspect"]).to eq 1.5
-      expect(media.file.meta["small"]["width"]).to eq 490
-      expect(media.file.meta["small"]["height"]).to eq 327
-      expect(media.file.meta["small"]["aspect"]).to eq 490.0 / 327
+      expect(media.file.meta["small"]["width"]).to eq 588
+      expect(media.file.meta["small"]["height"]).to eq 392
+      expect(media.file.meta["small"]["aspect"]).to eq 1.5
     end
 
     it 'gives the file a random name' do
-- 
cgit 


From 36b0ff57b7699db3d6acaa969a6c1491d1b9f9e3 Mon Sep 17 00:00:00 2001
From: Roni Laukkarinen <roni@laukkarinen.info>
Date: Tue, 8 Nov 2022 17:35:42 +0200
Subject: Fix grammar (#20106)

---
 spec/models/account_spec.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'spec/models')

diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 467d41836..75e0235b8 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -755,7 +755,7 @@ RSpec.describe Account, type: :model do
         expect(account).to model_have_error_on_field(:username)
       end
 
-      it 'is invalid if the username is longer then 30 characters' do
+      it 'is invalid if the username is longer than 30 characters' do
         account = Fabricate.build(:account, username: Faker::Lorem.characters(number: 31))
         account.valid?
         expect(account).to model_have_error_on_field(:username)
@@ -801,7 +801,7 @@ RSpec.describe Account, type: :model do
         expect(account).to model_have_error_on_field(:username)
       end
 
-      it 'is valid even if the username is longer then 30 characters' do
+      it 'is valid even if the username is longer than 30 characters' do
         account = Fabricate.build(:account, domain: 'domain', username: Faker::Lorem.characters(number: 31))
         account.valid?
         expect(account).not_to model_have_error_on_field(:username)
-- 
cgit 


From 6ba52306f9d2611a06326445230b54c156c37e53 Mon Sep 17 00:00:00 2001
From: luzpaz <luzpaz@users.noreply.github.com>
Date: Tue, 8 Nov 2022 11:32:03 -0500
Subject: Fix typos (#19849)

Found via `codespell -q 3 -S ./yarn.lock,./CHANGELOG.md,./AUTHORS.md,./config/locales,./app/javascript/mastodon/locales -L ba,followings,keypair,medias,pattens,pixelx,rememberable,ro,te`
---
 spec/lib/webfinger_resource_spec.rb | 2 +-
 spec/models/account_spec.rb         | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'spec/models')

diff --git a/spec/lib/webfinger_resource_spec.rb b/spec/lib/webfinger_resource_spec.rb
index 236e9f3e2..5c7f475d6 100644
--- a/spec/lib/webfinger_resource_spec.rb
+++ b/spec/lib/webfinger_resource_spec.rb
@@ -50,7 +50,7 @@ describe WebfingerResource do
       end
 
       it 'finds the username in a mixed case http route' do
-        resource = 'HTTp://exAMPLEe.com/users/alice'
+        resource = 'HTTp://exAMPLe.com/users/alice'
 
         result = WebfingerResource.new(resource).username
         expect(result).to eq 'alice'
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 75e0235b8..edae05f9d 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -255,7 +255,7 @@ RSpec.describe Account, type: :model do
         Fabricate(:status, reblog: original_status, account: author)
       end
 
-      it 'is is true when this account has favourited it' do
+      it 'is true when this account has favourited it' do
         Fabricate(:favourite, status: original_reblog, account: subject)
 
         expect(subject.favourited?(original_status)).to eq true
@@ -267,7 +267,7 @@ RSpec.describe Account, type: :model do
     end
 
     context 'when the status is an original status' do
-      it 'is is true when this account has favourited it' do
+      it 'is true when this account has favourited it' do
         Fabricate(:favourite, status: original_status, account: subject)
 
         expect(subject.favourited?(original_status)).to eq true
-- 
cgit 


From e98833748e80275a88560155a0b912667dd2d70b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 9 Nov 2022 08:24:21 +0100
Subject: Fix being able to spoof link verification (#20217)

- Change verification to happen in `default` queue
- Change verification worker to only be queued if there's something to do
- Add `link` tags from metadata fields to page header of profiles
---
 app/models/account.rb                              |  44 +------
 app/models/account/field.rb                        |  73 ++++++++++++
 .../activitypub/process_account_service.rb         |   2 +-
 app/services/update_account_service.rb             |   2 +-
 app/views/accounts/show.html.haml                  |   3 +
 app/workers/verify_account_links_worker.rb         |   5 +-
 spec/models/account/field_spec.rb                  | 130 +++++++++++++++++++++
 7 files changed, 211 insertions(+), 48 deletions(-)
 create mode 100644 app/models/account/field.rb
 create mode 100644 spec/models/account/field_spec.rb

(limited to 'spec/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index 3647b8225..be1968fa6 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -295,7 +295,7 @@ class Account < ApplicationRecord
 
   def fields
     (self[:fields] || []).map do |f|
-      Field.new(self, f)
+      Account::Field.new(self, f)
     rescue
       nil
     end.compact
@@ -401,48 +401,6 @@ class Account < ApplicationRecord
     requires_review? && !requested_review?
   end
 
-  class Field < ActiveModelSerializers::Model
-    attributes :name, :value, :verified_at, :account
-
-    def initialize(account, attributes)
-      @original_field = attributes
-      string_limit = account.local? ? 255 : 2047
-      super(
-        account:     account,
-        name:        attributes['name'].strip[0, string_limit],
-        value:       attributes['value'].strip[0, string_limit],
-        verified_at: attributes['verified_at']&.to_datetime,
-      )
-    end
-
-    def verified?
-      verified_at.present?
-    end
-
-    def value_for_verification
-      @value_for_verification ||= begin
-        if account.local?
-          value
-        else
-          ActionController::Base.helpers.strip_tags(value)
-        end
-      end
-    end
-
-    def verifiable?
-      value_for_verification.present? && value_for_verification.start_with?('http://', 'https://')
-    end
-
-    def mark_verified!
-      self.verified_at = Time.now.utc
-      @original_field['verified_at'] = verified_at
-    end
-
-    def to_h
-      { name: name, value: value, verified_at: verified_at }
-    end
-  end
-
   class << self
     DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze
     TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
diff --git a/app/models/account/field.rb b/app/models/account/field.rb
new file mode 100644
index 000000000..4e0fd9230
--- /dev/null
+++ b/app/models/account/field.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+class Account::Field < ActiveModelSerializers::Model
+  MAX_CHARACTERS_LOCAL  = 255
+  MAX_CHARACTERS_COMPAT = 2_047
+
+  attributes :name, :value, :verified_at, :account
+
+  def initialize(account, attributes)
+    # Keeping this as reference allows us to update the field on the account
+    # from methods in this class, so that changes can be saved.
+    @original_field = attributes
+    @account        = account
+
+    super(
+      name:        sanitize(attributes['name']),
+      value:       sanitize(attributes['value']),
+      verified_at: attributes['verified_at']&.to_datetime,
+    )
+  end
+
+  def verified?
+    verified_at.present?
+  end
+
+  def value_for_verification
+    @value_for_verification ||= begin
+      if account.local?
+        value
+      else
+        extract_url_from_html
+      end
+    end
+  end
+
+  def verifiable?
+    value_for_verification.present? && /\A#{FetchLinkCardService::URL_PATTERN}\z/.match?(value_for_verification)
+  end
+
+  def requires_verification?
+    !verified? && verifiable?
+  end
+
+  def mark_verified!
+    @original_field['verified_at'] = self.verified_at = Time.now.utc
+  end
+
+  def to_h
+    { name: name, value: value, verified_at: verified_at }
+  end
+
+  private
+
+  def sanitize(str)
+    str.strip[0, character_limit]
+  end
+
+  def character_limit
+    account.local? ? MAX_CHARACTERS_LOCAL : MAX_CHARACTERS_COMPAT
+  end
+
+  def extract_url_from_html
+    doc = Nokogiri::HTML(value).at_xpath('//body')
+
+    return if doc.children.size > 1
+
+    element = doc.children.first
+
+    return if element.name != 'a' || element['href'] != element.text
+
+    element['href']
+  end
+end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 3834d79cc..99bcb3835 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -40,7 +40,7 @@ class ActivityPub::ProcessAccountService < BaseService
     unless @options[:only_key] || @account.suspended?
       check_featured_collection! if @account.featured_collection_url.present?
       check_featured_tags_collection! if @json['featuredTags'].present?
-      check_links! unless @account.fields.empty?
+      check_links! if @account.fields.any?(&:requires_verification?)
     end
 
     @account
diff --git a/app/services/update_account_service.rb b/app/services/update_account_service.rb
index 77f794e17..71976ab00 100644
--- a/app/services/update_account_service.rb
+++ b/app/services/update_account_service.rb
@@ -28,7 +28,7 @@ class UpdateAccountService < BaseService
   end
 
   def check_links(account)
-    VerifyAccountLinksWorker.perform_async(account.id)
+    VerifyAccountLinksWorker.perform_async(account.id) if account.fields.any?(&:requires_verification?)
   end
 
   def process_hashtags(account)
diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index a51dcd7be..e8fd27e10 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -8,6 +8,9 @@
   %link{ rel: 'alternate', type: 'application/rss+xml', href: @rss_url }/
   %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@account) }/
 
+  - @account.fields.select(&:verifiable?).each do |field|
+    %link{ rel: 'me', type: 'text/html', href: field.value }/
+
   = opengraph 'og:type', 'profile'
   = render 'og', account: @account, url: short_account_url(@account, only_path: false)
 
diff --git a/app/workers/verify_account_links_worker.rb b/app/workers/verify_account_links_worker.rb
index 8114d59be..f606e6c26 100644
--- a/app/workers/verify_account_links_worker.rb
+++ b/app/workers/verify_account_links_worker.rb
@@ -3,14 +3,13 @@
 class VerifyAccountLinksWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', retry: false, lock: :until_executed
+  sidekiq_options queue: 'default', retry: false, lock: :until_executed
 
   def perform(account_id)
     account = Account.find(account_id)
 
     account.fields.each do |field|
-      next unless !field.verified? && field.verifiable?
-      VerifyLinkService.new.call(field)
+      VerifyLinkService.new.call(field) if field.requires_verification?
     end
 
     account.save! if account.changed?
diff --git a/spec/models/account/field_spec.rb b/spec/models/account/field_spec.rb
new file mode 100644
index 000000000..7d61a2c62
--- /dev/null
+++ b/spec/models/account/field_spec.rb
@@ -0,0 +1,130 @@
+require 'rails_helper'
+
+RSpec.describe Account::Field, type: :model do
+  describe '#verified?' do
+    let(:account) { double('Account', local?: true) }
+
+    subject { described_class.new(account, 'name' => 'Foo', 'value' => 'Bar', 'verified_at' => verified_at) }
+
+    context 'when verified_at is set' do
+      let(:verified_at) { Time.now.utc.iso8601 }
+
+      it 'returns true' do
+        expect(subject.verified?).to be true
+      end
+    end
+
+    context 'when verified_at is not set' do
+      let(:verified_at) { nil }
+
+      it 'returns false' do
+        expect(subject.verified?).to be false
+      end
+    end
+  end
+
+  describe '#mark_verified!' do
+    let(:account) { double('Account', local?: true) }
+    let(:original_hash) { { 'name' => 'Foo', 'value' => 'Bar' } }
+
+    subject { described_class.new(account, original_hash) }
+
+    before do
+      subject.mark_verified!
+    end
+
+    it 'updates verified_at' do
+      expect(subject.verified_at).to_not be_nil
+    end
+
+    it 'updates original hash' do
+      expect(original_hash['verified_at']).to_not be_nil
+    end
+  end
+
+  describe '#verifiable?' do
+    let(:account) { double('Account', local?: local) }
+
+    subject { described_class.new(account, 'name' => 'Foo', 'value' => value) }
+
+    context 'for local accounts' do
+      let(:local) { true }
+
+      context 'for a URL with misleading authentication' do
+        let(:value) { 'https://spacex.com                                                                                            @h.43z.one' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+
+      context 'for a URL' do
+        let(:value) { 'https://example.com' }
+
+        it 'returns true' do
+          expect(subject.verifiable?).to be true
+        end
+      end
+
+      context 'for text that is not a URL' do
+        let(:value) { 'Hello world' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+
+      context 'for text that contains a URL' do
+        let(:value) { 'Hello https://example.com world' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+    end
+
+    context 'for remote accounts' do
+      let(:local) { false }
+
+      context 'for a link' do
+        let(:value) { '<a href="https://www.patreon.com/mastodon" target="_blank" rel="nofollow noopener noreferrer me"><span class="invisible">https://www.</span><span class="">patreon.com/mastodon</span><span class="invisible"></span></a>' }
+
+        it 'returns true' do
+          expect(subject.verifiable?).to be true
+        end
+      end
+
+      context 'for a link with misleading authentication' do
+        let(:value) { '<a href="https://google.com                                                                                            @h.43z.one" target="_blank" rel="nofollow noopener noreferrer me"><span class="invisible">https://</span><span class="">google.com</span><span class="invisible">                                                                                            @h.43z.one</span></a>' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+
+      context 'for HTML that has more than just a link' do
+        let(:value) { '<a href="https://google.com" target="_blank" rel="nofollow noopener noreferrer me"><span class="invisible">https://</span><span class="">google.com</span><span class="invisible"></span></a>                                                                                            @h.43z.one' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+
+      context 'for a link with different visible text' do
+        let(:value) { '<a href="https://google.com/bar">https://example.com/foo</a>' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+
+      context 'for text that is a URL but is not linked' do
+        let(:value) { 'https://example.com/foo' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+    end
+  end
+end
-- 
cgit 


From 9965a23b043b0ab511e083c44acda891ea441859 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 10 Nov 2022 06:27:45 +0100
Subject: Change link verification to ignore IDN domains (#20295)

Fix #3833
---
 app/models/account/field.rb       | 16 +++++++++++++++-
 spec/models/account/field_spec.rb |  8 ++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

(limited to 'spec/models')

diff --git a/app/models/account/field.rb b/app/models/account/field.rb
index 4e0fd9230..d74f90b2b 100644
--- a/app/models/account/field.rb
+++ b/app/models/account/field.rb
@@ -3,6 +3,7 @@
 class Account::Field < ActiveModelSerializers::Model
   MAX_CHARACTERS_LOCAL  = 255
   MAX_CHARACTERS_COMPAT = 2_047
+  ACCEPTED_SCHEMES      = %w(http https).freeze
 
   attributes :name, :value, :verified_at, :account
 
@@ -34,7 +35,20 @@ class Account::Field < ActiveModelSerializers::Model
   end
 
   def verifiable?
-    value_for_verification.present? && /\A#{FetchLinkCardService::URL_PATTERN}\z/.match?(value_for_verification)
+    return false if value_for_verification.blank?
+
+    # This is slower than checking through a regular expression, but we
+    # need to confirm that it's not an IDN domain.
+
+    parsed_url = Addressable::URI.parse(value_for_verification)
+
+    ACCEPTED_SCHEMES.include?(parsed_url.scheme) &&
+      parsed_url.user.nil? &&
+      parsed_url.password.nil? &&
+      parsed_url.host.present? &&
+      parsed_url.normalized_host == parsed_url.host
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+    false
   end
 
   def requires_verification?
diff --git a/spec/models/account/field_spec.rb b/spec/models/account/field_spec.rb
index 7d61a2c62..fcb2a884a 100644
--- a/spec/models/account/field_spec.rb
+++ b/spec/models/account/field_spec.rb
@@ -66,6 +66,14 @@ RSpec.describe Account::Field, type: :model do
         end
       end
 
+      context 'for an IDN URL' do
+        let(:value) { 'http://twitter.com∕dougallj∕status∕1590357240443437057.ê.cc/twitter.html' }
+
+        it 'returns false' do
+          expect(subject.verifiable?).to be false
+        end
+      end
+
       context 'for text that is not a URL' do
         let(:value) { 'Hello world' }
 
-- 
cgit