Add trending links (#16917)

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests

16 files changed, 301 insertions, 21 deletions

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index d2ec122a4..3aeecb4ec 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -129,8 +129,6 @@ class ActivityPub::Activity
   end
 
   def crawl_links(status)
-    return if status.spoiler_text?
-
     # Spread out crawling randomly to avoid DDoSing the link
     LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 9f778ffb9..6c5d88d18 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
         visibility: visibility_from_audience
       )
 
-      original_status.tags.each do |tag|
-        tag.use!(@account)
-      end
+      Trends.tags.register(@status)
+      Trends.links.register(@status)
 
       distribute(@status)
     end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 4c13a80a6..8a0dc9d33 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
-      tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
+      tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
     end
 
+    # If we're processing an old status, this may register tags as being used now
+    # as opposed to when the status was really published, but this is probably
+    # not a big deal
+    Trends.tags.register(status)
+
     @mentions.each do |mention|
       mention.status = status
       mention.save
diff --git a/app/lib/admin/metrics/dimension.rb b/app/lib/admin/metrics/dimension.rb
index 279539f68..d8392ddfc 100644
--- a/app/lib/admin/metrics/dimension.rb
+++ b/app/lib/admin/metrics/dimension.rb
@@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
     servers: Admin::Metrics::Dimension::ServersDimension,
     space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
     software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
+    tag_servers: Admin::Metrics::Dimension::TagServersDimension,
+    tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
   }.freeze
 
-  def self.retrieve(dimension_keys, start_at, end_at, limit)
-    Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact
+  def self.retrieve(dimension_keys, start_at, end_at, limit, params)
+    Array(dimension_keys).map do |key|
+      klass = DIMENSIONS[key.to_sym]
+      klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
+    end.compact
   end
 end
diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb
index 8ed8d7683..5872c22cb 100644
--- a/app/lib/admin/metrics/dimension/base_dimension.rb
+++ b/app/lib/admin/metrics/dimension/base_dimension.rb
@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::BaseDimension
-  def initialize(start_at, end_at, limit)
+  def self.with_params?
+    false
+  end
+
+  def initialize(start_at, end_at, limit, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
     @limit    = limit&.to_i
+    @params   = params
   end
 
   def key
@@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
   protected
 
   def time_period
-    (@start_at...@end_at)
+    (@start_at..@end_at)
+  end
+
+  def params
+    raise NotImplementedError
   end
 end
diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb
index 2d0ac124e..a6aaf5d21 100644
--- a/app/lib/admin/metrics/dimension/languages_dimension.rb
+++ b/app/lib/admin/metrics/dimension/languages_dimension.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
+  include LanguagesHelper
+
   def key
     'languages'
   end
@@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
 
     rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
 
-    rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } }
+    rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
   end
 end
diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
new file mode 100644
index 000000000..1cfa07478
--- /dev/null
+++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
+  include LanguagesHelper
+
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_languages'
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
+      FROM statuses
+      INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
+      WHERE statuses_tags.tag_id = $1
+        AND statuses.id BETWEEN $2 AND $3
+      GROUP BY COALESCE(statuses.language, 'und')
+      ORDER BY count(*) DESC
+      LIMIT $4
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
+
+    rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
+  end
+
+  private
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
new file mode 100644
index 000000000..12c5980d7
--- /dev/null
+++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_servers'
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT accounts.domain, count(*) AS value
+      FROM statuses
+      INNER JOIN accounts ON accounts.id = statuses.account_id
+      INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
+      WHERE statuses_tags.tag_id = $1
+        AND statuses.id BETWEEN $2 AND $3
+      GROUP BY accounts.domain
+      ORDER BY count(*) DESC
+      LIMIT $4
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
+
+    rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
+  end
+
+  private
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/measure.rb b/app/lib/admin/metrics/measure.rb
index 5cebf0331..a839498a1 100644
--- a/app/lib/admin/metrics/measure.rb
+++ b/app/lib/admin/metrics/measure.rb
@@ -7,9 +7,15 @@ class Admin::Metrics::Measure
     interactions: Admin::Metrics::Measure::InteractionsMeasure,
     opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
     resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
+    tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
+    tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
+    tag_servers: Admin::Metrics::Measure::TagServersMeasure,
   }.freeze
 
-  def self.retrieve(measure_keys, start_at, end_at)
-    Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact
+  def self.retrieve(measure_keys, start_at, end_at, params)
+    Array(measure_keys).map do |key|
+      klass = MEASURES[key.to_sym]
+      klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
+    end.compact
   end
 end
diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb
index ac022eb9d..513189780 100644
--- a/app/lib/admin/metrics/measure/active_users_measure.rb
+++ b/app/lib/admin/metrics/measure/active_users_measure.rb
@@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
   end
 
   def time_period
-    (@start_at.to_date...@end_at.to_date)
+    (@start_at.to_date..@end_at.to_date)
   end
 
   def previous_time_period
-    ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
   end
 end
diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb
index 4c336a69e..0107ffd9c 100644
--- a/app/lib/admin/metrics/measure/base_measure.rb
+++ b/app/lib/admin/metrics/measure/base_measure.rb
@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Measure::BaseMeasure
-  def initialize(start_at, end_at)
+  def self.with_params?
+    false
+  end
+
+  def initialize(start_at, end_at, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
+    @params   = params
   end
 
   def key
@@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
   protected
 
   def time_period
-    (@start_at...@end_at)
+    (@start_at..@end_at)
   end
 
   def previous_time_period
-    ((@start_at - length_of_period)...(@end_at - length_of_period))
+    ((@start_at - length_of_period)..(@end_at - length_of_period))
   end
 
   def length_of_period
     @length_of_period ||= @end_at - @start_at
   end
+
+  def params
+    raise NotImplementedError
+  end
 end
diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb
index 9a4ef6d63..b928fdb8f 100644
--- a/app/lib/admin/metrics/measure/interactions_measure.rb
+++ b/app/lib/admin/metrics/measure/interactions_measure.rb
@@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
   end
 
   def time_period
-    (@start_at.to_date...@end_at.to_date)
+    (@start_at.to_date..@end_at.to_date)
   end
 
   def previous_time_period
-    ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
   end
 end
diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb
new file mode 100644
index 000000000..ef773081b
--- /dev/null
+++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_accounts'
+  end
+
+  def total
+    tag.history.aggregate(time_period).accounts
+  end
+
+  def previous_total
+    tag.history.aggregate(previous_time_period).accounts
+  end
+
+  def data
+    time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def time_period
+    (@start_at.to_date..@end_at.to_date)
+  end
+
+  def previous_time_period
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb
new file mode 100644
index 000000000..8c3e0551a
--- /dev/null
+++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_servers'
+  end
+
+  def total
+    tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
+  end
+
+  def previous_total
+    tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT axis.*, (
+        SELECT count(*) AS value
+        FROM statuses
+        WHERE statuses.id BETWEEN $1 AND $2
+          AND date_trunc('day', statuses.created_at)::date = axis.day
+      )
+      FROM (
+        SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
+      ) as axis
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
+
+    rows.map { |row| { date: row['day'], value: row['value'].to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb
new file mode 100644
index 000000000..b7667bc6c
--- /dev/null
+++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_uses'
+  end
+
+  def total
+    tag.history.aggregate(time_period).uses
+  end
+
+  def previous_total
+    tag.history.aggregate(previous_time_period).uses
+  end
+
+  def data
+    time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def time_period
+    (@start_at.to_date..@end_at.to_date)
+  end
+
+  def previous_time_period
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end
diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index 8b38e8d0c..56ad0717b 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -4,6 +4,11 @@ class LinkDetailsExtractor
   include ActionView::Helpers::TagHelper
 
   class StructuredData
+    SUPPORTED_TYPES = %w(
+      NewsArticle
+      WebPage
+    ).freeze
+
     def initialize(data)
       @data = data
     end
@@ -16,6 +21,14 @@ class LinkDetailsExtractor
       json['description']
     end
 
+    def language
+      json['inLanguage']
+    end
+
+    def type
+      json['@type']
+    end
+
     def image
       obj = first_of_value(json['image'])
 
@@ -44,6 +57,10 @@ class LinkDetailsExtractor
       publisher['name']
     end
 
+    def publisher_logo
+      publisher.dig('logo', 'url')
+    end
+
     private
 
     def author
@@ -58,8 +75,12 @@ class LinkDetailsExtractor
       arr.is_a?(Array) ? arr.first : arr
     end
 
+    def root_array(root)
+      root.is_a?(Array) ? root : [root]
+    end
+
     def json
-      @json ||= first_of_value(Oj.load(@data))
+      @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
     end
   end
 
@@ -75,6 +96,7 @@ class LinkDetailsExtractor
       description: description || '',
       image_remote_url: image,
       type: type,
+      link_type: link_type,
       width: width || 0,
       height: height || 0,
       html: html || '',
@@ -83,6 +105,7 @@ class LinkDetailsExtractor
       author_name: author_name || '',
       author_url: author_url || '',
       embed_url: embed_url || '',
+      language: language,
     }
   end
 
@@ -90,6 +113,14 @@ class LinkDetailsExtractor
     player_url.present? ? :video : :link
   end
 
+  def link_type
+    if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
+      :article
+    else
+      :unknown
+    end
+  end
+
   def html
     player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
   end
@@ -138,6 +169,14 @@ class LinkDetailsExtractor
     valid_url_or_nil(opengraph_tag('twitter:player:stream'))
   end
 
+  def language
+    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
+  end
+
+  def icon
+    valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
+  end
+
   private
 
   def player_url
@@ -162,6 +201,14 @@ class LinkDetailsExtractor
     nil
   end
 
+  def valid_locale_or_nil(str)
+    return nil if str.blank?
+
+    code,  = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
+    locale = ISO_639.find(code)
+    locale&.alpha2
+  end
+
   def link_tag(name)
     document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
   end