Add trending links (#16917)

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests

16 files changed, 775 insertions, 176 deletions

diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb
index 0a9551ec2..0f78c1a54 100644
--- a/app/models/account_statuses_cleanup_policy.rb
+++ b/app/models/account_statuses_cleanup_policy.rb
@@ -4,8 +4,8 @@
 #
 # Table name: account_statuses_cleanup_policies
 #
-#  id                 :bigint           not null, primary key
-#  account_id         :bigint           not null
+#  id                 :bigint(8)        not null, primary key
+#  account_id         :bigint(8)        not null
 #  enabled            :boolean          default(TRUE), not null
 #  min_status_age     :integer          default(1209600), not null
 #  keep_direct        :boolean          default(TRUE), not null
diff --git a/app/models/form/preview_card_batch.rb b/app/models/form/preview_card_batch.rb
new file mode 100644
index 000000000..5f6e6522a
--- /dev/null
+++ b/app/models/form/preview_card_batch.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Form::PreviewCardBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :preview_card_ids, :action, :current_account, :precision
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'approve_all'
+      approve_all!
+    when 'reject'
+      reject!
+    when 'reject_all'
+      reject_all!
+    end
+  end
+
+  private
+
+  def preview_cards
+    @preview_cards ||= PreviewCard.where(id: preview_card_ids)
+  end
+
+  def preview_card_providers
+    @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
+  end
+
+  def approve!
+    preview_cards.each { |preview_card| authorize(preview_card, :update?) }
+    preview_cards.update_all(trendable: true)
+  end
+
+  def approve_all!
+    preview_card_providers.each do |provider|
+      authorize(provider, :update?)
+      provider.update(trendable: true, reviewed_at: action_time)
+    end
+
+    # Reset any individual overrides
+    preview_cards.update_all(trendable: nil)
+  end
+
+  def reject!
+    preview_cards.each { |preview_card| authorize(preview_card, :update?) }
+    preview_cards.update_all(trendable: false)
+  end
+
+  def reject_all!
+    preview_card_providers.each do |provider|
+      authorize(provider, :update?)
+      provider.update(trendable: false, reviewed_at: action_time)
+    end
+
+    # Reset any individual overrides
+    preview_cards.update_all(trendable: nil)
+  end
+
+  def action_time
+    @action_time ||= Time.now.utc
+  end
+end
diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/form/preview_card_provider_batch.rb
new file mode 100644
index 000000000..e6ab3d8fa
--- /dev/null
+++ b/app/models/form/preview_card_provider_batch.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Form::PreviewCardProviderBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :preview_card_provider_ids, :action, :current_account
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'reject'
+      reject!
+    end
+  end
+
+  private
+
+  def preview_card_providers
+    PreviewCardProvider.where(id: preview_card_provider_ids)
+  end
+
+  def approve!
+    preview_card_providers.each { |provider| authorize(provider, :update?) }
+    preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
+  end
+
+  def reject!
+    preview_card_providers.each { |provider| authorize(provider, :update?) }
+    preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
+  end
+end
diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb
index fd517a1a6..b9330745f 100644
--- a/app/models/form/tag_batch.rb
+++ b/app/models/form/tag_batch.rb
@@ -23,11 +23,15 @@ class Form::TagBatch
 
   def approve!
     tags.each { |tag| authorize(tag, :update?) }
-    tags.update_all(trendable: true, reviewed_at: Time.now.utc)
+    tags.update_all(trendable: true, reviewed_at: action_time)
   end
 
   def reject!
     tags.each { |tag| authorize(tag, :update?) }
-    tags.update_all(trendable: false, reviewed_at: Time.now.utc)
+    tags.update_all(trendable: false, reviewed_at: action_time)
+  end
+
+  def action_time
+    @action_time ||= Time.now.utc
   end
 end
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index bca3a3ce8..f2ab8ecab 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -24,6 +24,11 @@
 #  embed_url                    :string           default(""), not null
 #  image_storage_schema_version :integer
 #  blurhash                     :string
+#  language                     :string
+#  max_score                    :float
+#  max_score_at                 :datetime
+#  trendable                    :boolean
+#  link_type                    :integer
 #
 
 class PreviewCard < ApplicationRecord
@@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord
   self.inheritance_column = false
 
   enum type: [:link, :photo, :video, :rich]
+  enum link_type: [:unknown, :article]
 
   has_and_belongs_to_many :statuses
 
@@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord
 
   before_save :extract_dimensions, if: :link?
 
+  def appropriate_for_trends?
+    link? && article? && title.present? && description.present? && image.present? && provider_name.present?
+  end
+
+  def domain
+    @domain ||= Addressable::URI.parse(url).normalized_host
+  end
+
+  def provider
+    @provider ||= PreviewCardProvider.matching_domain(domain)
+  end
+
+  def trendable?
+    if attributes['trendable'].nil?
+      provider&.trendable?
+    else
+      attributes['trendable']
+    end
+  end
+
+  def requires_review_notification?
+    attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?)
+  end
+
+  attr_writer :provider
+
   def local?
     false
   end
@@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord
     save!
   end
 
+  def history
+    @history ||= Trends::History.new('links', id)
+  end
+
   class << self
     private
 
-    # rubocop:disable Naming/MethodParameterName
-    def image_styles(f)
+    def image_styles(file)
       styles = {
         original: {
           geometry: '400x400>',
@@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord
         },
       }
 
-      styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif'
+      styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif'
       styles
     end
-    # rubocop:enable Naming/MethodParameterName
   end
 
   private
diff --git a/app/models/preview_card_filter.rb b/app/models/preview_card_filter.rb
new file mode 100644
index 000000000..8dda9989c
--- /dev/null
+++ b/app/models/preview_card_filter.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class PreviewCardFilter
+  KEYS = %i(
+    trending
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = PreviewCard.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'trending'
+      trending_scope(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+
+  def trending_scope(value)
+    ids = begin
+      case value.to_s
+      when 'allowed'
+        Trends.links.currently_trending_ids(true, -1)
+      else
+        Trends.links.currently_trending_ids(false, -1)
+      end
+    end
+
+    if ids.empty?
+      PreviewCard.none
+    else
+      PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
+    end
+  end
+end
diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb
new file mode 100644
index 000000000..15b24e2bd
--- /dev/null
+++ b/app/models/preview_card_provider.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: preview_card_providers
+#
+#  id                  :bigint(8)        not null, primary key
+#  domain              :string           default(""), not null
+#  icon_file_name      :string
+#  icon_content_type   :string
+#  icon_file_size      :bigint(8)
+#  icon_updated_at     :datetime
+#  trendable           :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+
+class PreviewCardProvider < ApplicationRecord
+  include DomainNormalizable
+  include Attachmentable
+
+  ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze
+  LIMIT = 1.megabyte
+
+  validates :domain, presence: true, uniqueness: true, domain: true
+
+  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
+  validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
+  remotable_attachment :icon, LIMIT
+
+  scope :trendable, -> { where(trendable: true) }
+  scope :not_trendable, -> { where(trendable: false) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :pending_review, -> { where(reviewed_at: nil) }
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def requires_review_notification?
+    requires_review? && !requested_review?
+  end
+
+  def self.matching_domain(domain)
+    segments = domain.split('.')
+    where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first
+  end
+end
diff --git a/app/models/preview_card_provider_filter.rb b/app/models/preview_card_provider_filter.rb
new file mode 100644
index 000000000..1e90d3c9d
--- /dev/null
+++ b/app/models/preview_card_provider_filter.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class PreviewCardProviderFilter
+  KEYS = %i(
+    status
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = PreviewCardProvider.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope.order(domain: :asc)
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'status'
+      status_scope(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+
+  def status_scope(value)
+    case value.to_s
+    when 'approved'
+      PreviewCardProvider.trendable
+    when 'rejected'
+      PreviewCardProvider.not_trendable
+    when 'pending_review'
+      PreviewCardProvider.pending_review
+    else
+      raise "Unknown status: #{value}"
+    end
+  end
+end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index dcce28391..f35d92b5d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -36,6 +36,7 @@ class Tag < ApplicationRecord
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
+  scope :not_trendable, -> { where(trendable: false) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
   scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
@@ -75,28 +76,12 @@ class Tag < ApplicationRecord
     requested_review_at.present?
   end
 
-  def use!(account, status: nil, at_time: Time.now.utc)
-    TrendingTags.record_use!(self, account, status: status, at_time: at_time)
-  end
-
-  def trending?
-    TrendingTags.trending?(self)
+  def requires_review_notification?
+    requires_review? && !requested_review?
   end
 
   def history
-    days = []
-
-    7.times do |i|
-      day = i.days.ago.beginning_of_day.to_i
-
-      days << {
-        day: day.to_s,
-        uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
-        accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
-      }
-    end
-
-    days
+    @history ||= Trends::History.new('tags', id)
   end
 
   class << self
diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb
index 85bfcbea5..ecdb52503 100644
--- a/app/models/tag_filter.rb
+++ b/app/models/tag_filter.rb
@@ -2,13 +2,8 @@
 
 class TagFilter
   KEYS = %i(
-    directory
-    reviewed
-    unreviewed
-    pending_review
-    popular
-    active
-    name
+    trending
+    status
   ).freeze
 
   attr_reader :params
@@ -18,7 +13,13 @@ class TagFilter
   end
 
   def results
-    scope = Tag.unscoped
+    scope = begin
+      if params[:status] == 'pending_review'
+        Tag.unscoped
+      else
+        trending_scope
+      end
+    end
 
     params.each do |key, value|
       next if key.to_s == 'page'
@@ -26,27 +27,40 @@ class TagFilter
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end
 
-    scope.order(id: :desc)
+    scope
   end
 
   private
 
   def scope_for(key, value)
     case key.to_s
-    when 'reviewed'
-      Tag.reviewed.order(reviewed_at: :desc)
-    when 'unreviewed'
-      Tag.unreviewed
-    when 'pending_review'
-      Tag.pending_review.order(requested_review_at: :desc)
-    when 'popular'
-      Tag.order('max_score DESC NULLS LAST')
-    when 'active'
-      Tag.order('last_status_at DESC NULLS LAST')
-    when 'name'
-      Tag.matches_name(value)
+    when 'status'
+      status_scope(value)
     else
       raise "Unknown filter: #{key}"
     end
   end
+
+  def trending_scope
+    ids = Trends.tags.currently_trending_ids(false, -1)
+
+    if ids.empty?
+      Tag.none
+    else
+      Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
+    end
+  end
+
+  def status_scope(value)
+    case value.to_s
+    when 'approved'
+      Tag.trendable
+    when 'rejected'
+      Tag.not_trendable
+    when 'pending_review'
+      Tag.pending_review
+    else
+      raise "Unknown status: #{value}"
+    end
+  end
 end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
deleted file mode 100644
index 31890b082..000000000
--- a/app/models/trending_tags.rb
+++ /dev/null
@@ -1,128 +0,0 @@
-# frozen_string_literal: true
-
-class TrendingTags
-  KEY                  = 'trending_tags'
-  EXPIRE_HISTORY_AFTER = 7.days.seconds
-  EXPIRE_TRENDS_AFTER  = 1.day.seconds
-  THRESHOLD            = 5
-  LIMIT                = 10
-  REVIEW_THRESHOLD     = 3
-  MAX_SCORE_COOLDOWN   = 2.days.freeze
-  MAX_SCORE_HALFLIFE   = 2.hours.freeze
-
-  class << self
-    include Redisable
-
-    def record_use!(tag, account, status: nil, at_time: Time.now.utc)
-      return unless tag.usable? && !account.silenced?
-
-      # Even if a tag is not allowed to trend, we still need to
-      # record the stats since they can be displayed in other places
-      increment_historical_use!(tag.id, at_time)
-      increment_unique_use!(tag.id, account.id, at_time)
-      increment_use!(tag.id, at_time)
-
-      # Only update when the tag was last used once every 12 hours
-      # and only if a status is given (lets use ignore reblogs)
-      tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
-    end
-
-    def update!(at_time = Time.now.utc)
-      tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
-      tags    = Tag.trendable.where(id: tag_ids.uniq)
-
-      # First pass to calculate scores and update the set
-
-      tags.each do |tag|
-        expected  = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
-        expected  = 1.0 if expected.zero?
-        observed  = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
-        max_time  = tag.max_score_at
-        max_score = tag.max_score
-        max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
-
-        score = begin
-          if expected > observed || observed < THRESHOLD
-            0
-          else
-            ((observed - expected)**2) / expected
-          end
-        end
-
-        if score > max_score
-          max_score = score
-          max_time  = at_time
-
-          # Not interested in triggering any callbacks for this
-          tag.update_columns(max_score: max_score, max_score_at: max_time)
-        end
-
-        decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
-
-        if decaying_score.zero?
-          redis.zrem(KEY, tag.id)
-        else
-          redis.zadd(KEY, decaying_score, tag.id)
-        end
-      end
-
-      users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
-
-      # Second pass to notify about previously unreviewed trends
-
-      tags.each do |tag|
-        current_rank              = redis.zrevrank(KEY, tag.id)
-        needs_review_notification = tag.requires_review? && !tag.requested_review?
-        rank_passes_threshold     = current_rank.present? && current_rank <= REVIEW_THRESHOLD
-
-        next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
-
-        tag.touch(:requested_review_at)
-
-        users_for_review.each do |user|
-          AdminMailer.new_trending_tag(user.account, tag).deliver_later!
-        end
-      end
-
-      # Trim older items
-
-      redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
-      redis.zremrangebyscore(KEY, '(0.3', '-inf')
-    end
-
-    def get(limit, filtered: true)
-      tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
-
-      tags = Tag.where(id: tag_ids)
-      tags = tags.trendable if filtered
-      tags = tags.index_by(&:id)
-
-      tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
-    end
-
-    def trending?(tag)
-      rank = redis.zrevrank(KEY, tag.id)
-      rank.present? && rank < LIMIT
-    end
-
-    private
-
-    def increment_historical_use!(tag_id, at_time)
-      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
-      redis.incrby(key, 1)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-
-    def increment_unique_use!(tag_id, account_id, at_time)
-      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
-      redis.pfadd(key, account_id)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-
-    def increment_use!(tag_id, at_time)
-      key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
-      redis.sadd(key, tag_id)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-  end
-end
diff --git a/app/models/trends.rb b/app/models/trends.rb
new file mode 100644
index 000000000..7dd3a9c87
--- /dev/null
+++ b/app/models/trends.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Trends
+  def self.table_name_prefix
+    'trends_'
+  end
+
+  def self.links
+    @links ||= Trends::Links.new
+  end
+
+  def self.tags
+    @tags ||= Trends::Tags.new
+  end
+
+  def self.refresh!
+    [links, tags].each(&:refresh)
+  end
+
+  def self.request_review!
+    [links, tags].each(&:request_review) if enabled?
+  end
+
+  def self.enabled?
+    Setting.trends
+  end
+end
diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb
new file mode 100644
index 000000000..b767dcb1a
--- /dev/null
+++ b/app/models/trends/base.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class Trends::Base
+  include Redisable
+
+  class_attribute :default_options
+
+  attr_reader :options
+
+  # @param [Hash] options
+  # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score
+  # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review
+  # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from
+  # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays
+  def initialize(options = {})
+    @options = self.class.default_options.merge(options)
+  end
+
+  def register(_status)
+    raise NotImplementedError
+  end
+
+  def add(*)
+    raise NotImplementedError
+  end
+
+  def refresh(*)
+    raise NotImplementedError
+  end
+
+  def request_review
+    raise NotImplementedError
+  end
+
+  def get(*)
+    raise NotImplementedError
+  end
+
+  def score(id)
+    redis.zscore("#{key_prefix}:all", id) || 0
+  end
+
+  def rank(id)
+    redis.zrevrank("#{key_prefix}:allowed", id)
+  end
+
+  def currently_trending_ids(allowed, limit)
+    redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
+  end
+
+  protected
+
+  def key_prefix
+    raise NotImplementedError
+  end
+
+  def recently_used_ids(at_time = Time.now.utc)
+    redis.smembers(used_key(at_time)).map(&:to_i)
+  end
+
+  def record_used_id(id, at_time = Time.now.utc)
+    redis.sadd(used_key(at_time), id)
+    redis.expire(used_key(at_time), 1.day.seconds)
+  end
+
+  def trim_older_items
+    redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
+    redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
+  end
+
+  def score_at_rank(rank)
+    redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
+  end
+
+  private
+
+  def used_key(at_time)
+    "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
+  end
+end
diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb
new file mode 100644
index 000000000..608e33792
--- /dev/null
+++ b/app/models/trends/history.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+class Trends::History
+  include Enumerable
+
+  class Aggregate
+    include Redisable
+
+    def initialize(prefix, id, date_range)
+      @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) }
+    end
+
+    def uses
+      redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum
+    end
+
+    def accounts
+      redis.pfcount(*@days.map { |day| day.key_for(:accounts) })
+    end
+  end
+
+  class Day
+    include Redisable
+
+    EXPIRE_AFTER = 14.days.seconds
+
+    def initialize(prefix, id, day)
+      @prefix = prefix
+      @id     = id
+      @day    = day.beginning_of_day
+    end
+
+    attr_reader :day
+
+    def accounts
+      redis.pfcount(key_for(:accounts))
+    end
+
+    def uses
+      redis.get(key_for(:uses))&.to_i || 0
+    end
+
+    def add(account_id)
+      redis.pipelined do
+        redis.incrby(key_for(:uses), 1)
+        redis.pfadd(key_for(:accounts), account_id)
+        redis.expire(key_for(:uses), EXPIRE_AFTER)
+        redis.expire(key_for(:accounts), EXPIRE_AFTER)
+      end
+    end
+
+    def as_json
+      { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s }
+    end
+
+    def key_for(suffix)
+      case suffix
+      when :accounts
+        "#{key_prefix}:#{suffix}"
+      when :uses
+        key_prefix
+      end
+    end
+
+    def key_prefix
+      "activity:#{@prefix}:#{@id}:#{day.to_i}"
+    end
+  end
+
+  def initialize(prefix, id)
+    @prefix = prefix
+    @id     = id
+  end
+
+  def get(date)
+    Day.new(@prefix, @id, date)
+  end
+
+  def add(account_id, at_time = Time.now.utc)
+    Day.new(@prefix, @id, at_time).add(account_id)
+  end
+
+  def aggregate(date_range)
+    Aggregate.new(@prefix, @id, date_range)
+  end
+
+  def each(&block)
+    if block_given?
+      (0...7).map { |i| block.call(get(i.days.ago)) }
+    else
+      to_enum(:each)
+    end
+  end
+
+  def as_json(*)
+    map(&:as_json)
+  end
+end
diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb
new file mode 100644
index 000000000..a0d65138b
--- /dev/null
+++ b/app/models/trends/links.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+class Trends::Links < Trends::Base
+  PREFIX = 'trending_links'
+
+  self.default_options = {
+    threshold: 15,
+    review_threshold: 10,
+    max_score_cooldown: 2.days.freeze,
+    max_score_halflife: 8.hours.freeze,
+  }
+
+  def register(status, at_time = Time.now.utc)
+    original_status = status.reblog? ? status.reblog : status
+
+    return unless original_status.public_visibility? && status.public_visibility? &&
+                  !original_status.account.silenced? && !status.account.silenced? &&
+                  !original_status.spoiler_text?
+
+    original_status.preview_cards.each do |preview_card|
+      add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends?
+    end
+  end
+
+  def add(preview_card, account_id, at_time = Time.now.utc)
+    preview_card.history.add(account_id, at_time)
+    record_used_id(preview_card.id, at_time)
+  end
+
+  def get(allowed, limit)
+    preview_card_ids = currently_trending_ids(allowed, limit)
+    preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
+    preview_card_ids.map { |id| preview_cards[id] }.compact
+  end
+
+  def refresh(at_time = Time.now.utc)
+    preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    calculate_scores(preview_cards, at_time)
+    trim_older_items
+  end
+
+  def request_review
+    preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
+
+    preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
+      next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
+
+      if preview_card.provider.nil?
+        preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
+      else
+        preview_card.provider.touch(:requested_review_at)
+      end
+
+      preview_card
+    end
+
+    return if preview_cards_requiring_review.empty?
+
+    User.staff.includes(:account).find_each do |user|
+      AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
+    end
+  end
+
+  protected
+
+  def key_prefix
+    PREFIX
+  end
+
+  private
+
+  def calculate_scores(preview_cards, at_time)
+    preview_cards.each do |preview_card|
+      expected  = preview_card.history.get(at_time - 1.day).accounts.to_f
+      expected  = 1.0 if expected.zero?
+      observed  = preview_card.history.get(at_time).accounts.to_f
+      max_time  = preview_card.max_score_at
+      max_score = preview_card.max_score
+      max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
+
+      score = begin
+        if expected > observed || observed < options[:threshold]
+          0
+        else
+          ((observed - expected)**2) / expected
+        end
+      end
+
+      if score > max_score
+        max_score = score
+        max_time  = at_time
+
+        # Not interested in triggering any callbacks for this
+        preview_card.update_columns(max_score: max_score, max_score_at: max_time)
+      end
+
+      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+
+      if decaying_score.zero?
+        redis.zrem("#{PREFIX}:all", preview_card.id)
+        redis.zrem("#{PREFIX}:allowed", preview_card.id)
+      else
+        redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
+
+        if preview_card.trendable?
+          redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
+        else
+          redis.zrem("#{PREFIX}:allowed", preview_card.id)
+        end
+      end
+    end
+  end
+
+  def would_be_trending?(id)
+    score(id) > score_at_rank(options[:review_threshold] - 1)
+  end
+end
diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb
new file mode 100644
index 000000000..13e0ab56b
--- /dev/null
+++ b/app/models/trends/tags.rb
@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+class Trends::Tags < Trends::Base
+  PREFIX = 'trending_tags'
+
+  self.default_options = {
+    threshold: 15,
+    review_threshold: 10,
+    max_score_cooldown: 2.days.freeze,
+    max_score_halflife: 4.hours.freeze,
+  }
+
+  def register(status, at_time = Time.now.utc)
+    original_status = status.reblog? ? status.reblog : status
+
+    return unless original_status.public_visibility? && status.public_visibility? &&
+                  !original_status.account.silenced? && !status.account.silenced?
+
+    original_status.tags.each do |tag|
+      add(tag, status.account_id, at_time) if tag.usable?
+    end
+  end
+
+  def add(tag, account_id, at_time = Time.now.utc)
+    tag.history.add(account_id, at_time)
+    record_used_id(tag.id, at_time)
+  end
+
+  def refresh(at_time = Time.now.utc)
+    tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    calculate_scores(tags, at_time)
+    trim_older_items
+  end
+
+  def get(allowed, limit)
+    tag_ids = currently_trending_ids(allowed, limit)
+    tags = Tag.where(id: tag_ids).index_by(&:id)
+    tag_ids.map { |id| tags[id] }.compact
+  end
+
+  def request_review
+    tags = Tag.where(id: currently_trending_ids(false, -1))
+
+    tags_requiring_review = tags.filter_map do |tag|
+      next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
+
+      tag.touch(:requested_review_at)
+      tag
+    end
+
+    return if tags_requiring_review.empty?
+
+    User.staff.includes(:account).find_each do |user|
+      AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
+    end
+  end
+
+  protected
+
+  def key_prefix
+    PREFIX
+  end
+
+  private
+
+  def calculate_scores(tags, at_time)
+    tags.each do |tag|
+      expected  = tag.history.get(at_time - 1.day).accounts.to_f
+      expected  = 1.0 if expected.zero?
+      observed  = tag.history.get(at_time).accounts.to_f
+      max_time  = tag.max_score_at
+      max_score = tag.max_score
+      max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
+
+      score = begin
+        if expected > observed || observed < options[:threshold]
+          0
+        else
+          ((observed - expected)**2) / expected
+        end
+      end
+
+      if score > max_score
+        max_score = score
+        max_time  = at_time
+
+        # Not interested in triggering any callbacks for this
+        tag.update_columns(max_score: max_score, max_score_at: max_time)
+      end
+
+      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+
+      if decaying_score.zero?
+        redis.zrem("#{PREFIX}:all", tag.id)
+        redis.zrem("#{PREFIX}:allowed", tag.id)
+      else
+        redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
+
+        if tag.trendable?
+          redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
+        else
+          redis.zrem("#{PREFIX}:allowed", tag.id)
+        end
+      end
+    end
+  end
+
+  def would_be_trending?(id)
+    score(id) > score_at_rank(options[:review_threshold] - 1)
+  end
+end