Merge remote-tracking branch 'glitch/main'

22 files changed, 205 insertions, 60 deletions

diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 1f9319290..12fad8da4 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -8,6 +8,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       original_status = status_from_object
 
       return reject_payload! if original_status.nil? || !announceable?(original_status)
+      return if requested_through_relay?
 
       @status = Status.find_by(account: @account, reblog: original_status)
 
diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb
index 5872c22cb..bd2e4ecec 100644
--- a/app/lib/admin/metrics/dimension/base_dimension.rb
+++ b/app/lib/admin/metrics/dimension/base_dimension.rb
@@ -1,23 +1,34 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::BaseDimension
+  CACHE_TTL = 5.minutes.freeze
+
   def self.with_params?
     false
   end
 
+  attr_reader :loaded
+
+  alias loaded? loaded
+
   def initialize(start_at, end_at, limit, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
     @limit    = limit&.to_i
     @params   = params
+    @loaded   = false
   end
 
   def key
     raise NotImplementedError
   end
 
+  def cache_key
+    ["metrics/dimension/#{key}", @start_at, @end_at, @limit, canonicalized_params].join(';')
+  end
+
   def data
-    raise NotImplementedError
+    load
   end
 
   def self.model_name
@@ -30,11 +41,28 @@ class Admin::Metrics::Dimension::BaseDimension
 
   protected
 
+  def load
+    unless loaded?
+      @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query }
+      @loaded = true
+    end
+
+    @values
+  end
+
+  def perform_query
+    raise NotImplementedError
+  end
+
   def time_period
     (@start_at..@end_at)
   end
 
   def params
-    raise NotImplementedError
+    {}
+  end
+
+  def canonicalized_params
+    params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';')
   end
 end
diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb
index 1cc5f4120..f1cf82cf2 100644
--- a/app/lib/admin/metrics/dimension/languages_dimension.rb
+++ b/app/lib/admin/metrics/dimension/languages_dimension.rb
@@ -7,7 +7,9 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
     'languages'
   end
 
-  def data
+  protected
+
+  def perform_query
     sql = <<-SQL.squish
       SELECT locale, count(*) AS value
       FROM users
diff --git a/app/lib/admin/metrics/dimension/servers_dimension.rb b/app/lib/admin/metrics/dimension/servers_dimension.rb
index 3e80b6625..91bcce655 100644
--- a/app/lib/admin/metrics/dimension/servers_dimension.rb
+++ b/app/lib/admin/metrics/dimension/servers_dimension.rb
@@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::ServersDimension < Admin::Metrics::Dimension::B
     'servers'
   end
 
-  def data
+  protected
+
+  def perform_query
     sql = <<-SQL.squish
       SELECT accounts.domain, count(*) AS value
       FROM statuses
diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb
index 34917404d..816615f99 100644
--- a/app/lib/admin/metrics/dimension/software_versions_dimension.rb
+++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb
@@ -7,12 +7,12 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim
     'software_versions'
   end
 
-  def data
+  protected
+
+  def perform_query
     [mastodon_version, ruby_version, postgresql_version, redis_version]
   end
 
-  private
-
   def mastodon_version
     value = Mastodon::Version.to_s
 
diff --git a/app/lib/admin/metrics/dimension/sources_dimension.rb b/app/lib/admin/metrics/dimension/sources_dimension.rb
index a9f061809..122807cdc 100644
--- a/app/lib/admin/metrics/dimension/sources_dimension.rb
+++ b/app/lib/admin/metrics/dimension/sources_dimension.rb
@@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::SourcesDimension < Admin::Metrics::Dimension::B
     'sources'
   end
 
-  def data
+  protected
+
+  def perform_query
     sql = <<-SQL.squish
       SELECT oauth_applications.name, count(*) AS value
       FROM users
diff --git a/app/lib/admin/metrics/dimension/space_usage_dimension.rb b/app/lib/admin/metrics/dimension/space_usage_dimension.rb
index aa00a2e18..5867c5bab 100644
--- a/app/lib/admin/metrics/dimension/space_usage_dimension.rb
+++ b/app/lib/admin/metrics/dimension/space_usage_dimension.rb
@@ -8,12 +8,12 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension
     'space_usage'
   end
 
-  def data
+  protected
+
+  def perform_query
     [postgresql_size, redis_size, media_size]
   end
 
-  private
-
   def postgresql_size
     value = ActiveRecord::Base.connection.execute('SELECT pg_database_size(current_database())').first['pg_database_size']
 
diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
index afbc8cde8..e1349c229 100644
--- a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
+++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
@@ -11,7 +11,9 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi
     'tag_languages'
   end
 
-  def data
+  protected
+
+  def perform_query
     sql = <<-SQL.squish
       SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
       FROM statuses
@@ -28,8 +30,6 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi
     rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } }
   end
 
-  private
-
   def params
     @params.permit(:id)
   end
diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
index 12c5980d7..7ddf3378c 100644
--- a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
+++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb
@@ -9,7 +9,9 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension
     'tag_servers'
   end
 
-  def data
+  protected
+
+  def perform_query
     sql = <<-SQL.squish
       SELECT accounts.domain, count(*) AS value
       FROM statuses
@@ -27,8 +29,6 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension
     rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
   end
 
-  private
-
   def params
     @params.permit(:id)
   end
diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb
index 513189780..e6f09d4bc 100644
--- a/app/lib/admin/metrics/measure/active_users_measure.rb
+++ b/app/lib/admin/metrics/measure/active_users_measure.rb
@@ -5,20 +5,20 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
     'active_users'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     activity_tracker.sum(time_period.first, time_period.last)
   end
 
-  def previous_total
+  def perform_previous_total_query
     activity_tracker.sum(previous_time_period.first, previous_time_period.last)
   end
 
-  def data
+  def perform_data_query
     activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } }
   end
 
-  protected
-
   def activity_tracker
     @activity_tracker ||= ActivityTracker.new('activity:logins', :unique)
   end
diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb
index 0107ffd9c..ed1df9c7d 100644
--- a/app/lib/admin/metrics/measure/base_measure.rb
+++ b/app/lib/admin/metrics/measure/base_measure.rb
@@ -1,14 +1,25 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Measure::BaseMeasure
+  CACHE_TTL = 5.minutes.freeze
+
   def self.with_params?
     false
   end
 
+  attr_reader :loaded
+
+  alias loaded? loaded
+
   def initialize(start_at, end_at, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
     @params   = params
+    @loaded   = false
+  end
+
+  def cache_key
+    ["metrics/measure/#{key}", @start_at, @end_at, canonicalized_params].join(';')
   end
 
   def key
@@ -16,15 +27,15 @@ class Admin::Metrics::Measure::BaseMeasure
   end
 
   def total
-    raise NotImplementedError
+    load[:total]
   end
 
   def previous_total
-    raise NotImplementedError
+    load[:previous_total]
   end
 
   def data
-    raise NotImplementedError
+    load[:data]
   end
 
   def self.model_name
@@ -37,6 +48,35 @@ class Admin::Metrics::Measure::BaseMeasure
 
   protected
 
+  def load
+    unless loaded?
+      @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_queries }.with_indifferent_access
+      @loaded = true
+    end
+
+    @values
+  end
+
+  def perform_queries
+    {
+      total: perform_total_query,
+      previous_total: perform_previous_total_query,
+      data: perform_data_query,
+    }
+  end
+
+  def perform_total_query
+    raise NotImplementedError
+  end
+
+  def perform_previous_total_query
+    raise NotImplementedError
+  end
+
+  def perform_data_query
+    raise NotImplementedError
+  end
+
   def time_period
     (@start_at..@end_at)
   end
@@ -50,6 +90,10 @@ class Admin::Metrics::Measure::BaseMeasure
   end
 
   def params
-    raise NotImplementedError
+    {}
+  end
+
+  def canonicalized_params
+    params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';')
   end
 end
diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb
index b928fdb8f..7a2b7e0fa 100644
--- a/app/lib/admin/metrics/measure/interactions_measure.rb
+++ b/app/lib/admin/metrics/measure/interactions_measure.rb
@@ -5,20 +5,20 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
     'interactions'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     activity_tracker.sum(time_period.first, time_period.last)
   end
 
-  def previous_total
+  def perform_previous_total_query
     activity_tracker.sum(previous_time_period.first, previous_time_period.last)
   end
 
-  def data
+  def perform_data_query
     activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } }
   end
 
-  protected
-
   def activity_tracker
     @activity_tracker ||= ActivityTracker.new('activity:interactions', :basic)
   end
diff --git a/app/lib/admin/metrics/measure/new_users_measure.rb b/app/lib/admin/metrics/measure/new_users_measure.rb
index b31679ad3..71191f1a2 100644
--- a/app/lib/admin/metrics/measure/new_users_measure.rb
+++ b/app/lib/admin/metrics/measure/new_users_measure.rb
@@ -5,15 +5,17 @@ class Admin::Metrics::Measure::NewUsersMeasure < Admin::Metrics::Measure::BaseMe
     'new_users'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     User.where(created_at: time_period).count
   end
 
-  def previous_total
+  def perform_previous_total_query
     User.where(created_at: previous_time_period).count
   end
 
-  def data
+  def perform_data_query
     sql = <<-SQL.squish
       SELECT axis.*, (
         WITH new_users AS (
diff --git a/app/lib/admin/metrics/measure/opened_reports_measure.rb b/app/lib/admin/metrics/measure/opened_reports_measure.rb
index 9acc2c33d..4b80a0c8c 100644
--- a/app/lib/admin/metrics/measure/opened_reports_measure.rb
+++ b/app/lib/admin/metrics/measure/opened_reports_measure.rb
@@ -5,15 +5,17 @@ class Admin::Metrics::Measure::OpenedReportsMeasure < Admin::Metrics::Measure::B
     'opened_reports'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     Report.where(created_at: time_period).count
   end
 
-  def previous_total
+  def perform_previous_total_query
     Report.where(created_at: previous_time_period).count
   end
 
-  def data
+  def perform_data_query
     sql = <<-SQL.squish
       SELECT axis.*, (
         WITH new_reports AS (
diff --git a/app/lib/admin/metrics/measure/resolved_reports_measure.rb b/app/lib/admin/metrics/measure/resolved_reports_measure.rb
index 00cb24f7e..4ab746c8f 100644
--- a/app/lib/admin/metrics/measure/resolved_reports_measure.rb
+++ b/app/lib/admin/metrics/measure/resolved_reports_measure.rb
@@ -5,15 +5,17 @@ class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure:
     'resolved_reports'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     Report.resolved.where(action_taken_at: time_period).count
   end
 
-  def previous_total
+  def perform_previous_total_query
     Report.resolved.where(action_taken_at: previous_time_period).count
   end
 
-  def data
+  def perform_data_query
     sql = <<-SQL.squish
       SELECT axis.*, (
         WITH resolved_reports AS (
diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb
index ef773081b..8f4512efe 100644
--- a/app/lib/admin/metrics/measure/tag_accounts_measure.rb
+++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb
@@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::Bas
     'tag_accounts'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     tag.history.aggregate(time_period).accounts
   end
 
-  def previous_total
+  def perform_previous_total_query
     tag.history.aggregate(previous_time_period).accounts
   end
 
-  def data
+  def perform_data_query
     time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
   end
 
-  protected
-
   def tag
     @tag ||= Tag.find(params[:id])
   end
diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb
index cc064f63f..11f229602 100644
--- a/app/lib/admin/metrics/measure/tag_servers_measure.rb
+++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb
@@ -9,15 +9,17 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base
     'tag_servers'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
   end
 
-  def previous_total
+  def perform_previous_total_query
     tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
   end
 
-  def data
+  def perform_data_query
     sql = <<-SQL.squish
       SELECT axis.*, (
         SELECT count(distinct accounts.domain) AS value
@@ -38,8 +40,6 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base
     rows.map { |row| { date: row['day'], value: row['value'].to_s } }
   end
 
-  protected
-
   def tag
     @tag ||= Tag.find(params[:id])
   end
diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb
index b7667bc6c..bce86b89f 100644
--- a/app/lib/admin/metrics/measure/tag_uses_measure.rb
+++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb
@@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMea
     'tag_uses'
   end
 
-  def total
+  protected
+
+  def perform_total_query
     tag.history.aggregate(time_period).uses
   end
 
-  def previous_total
+  def perform_previous_total_query
     tag.history.aggregate(previous_time_period).uses
   end
 
-  def data
+  def perform_data_query
     time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
   end
 
-  protected
-
   def tag
     @tag ||= Tag.find(params[:id])
   end
diff --git a/app/lib/admin/metrics/retention.rb b/app/lib/admin/metrics/retention.rb
index 0179a6e28..f6135ac1e 100644
--- a/app/lib/admin/metrics/retention.rb
+++ b/app/lib/admin/metrics/retention.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Retention
+  CACHE_TTL = 5.minutes.freeze
+
   class Cohort < ActiveModelSerializers::Model
     attributes :period, :frequency, :data
   end
@@ -9,13 +11,37 @@ class Admin::Metrics::Retention
     attributes :date, :rate, :value
   end
 
+  attr_reader :loaded
+
+  alias loaded? loaded
+
   def initialize(start_at, end_at, frequency)
     @start_at  = start_at&.to_date
     @end_at    = end_at&.to_date
     @frequency = %w(day month).include?(frequency) ? frequency : 'day'
+    @loaded    = false
+  end
+
+  def cache_key
+    ['metrics/retention', @start_at, @end_at, @frequency].join(';')
   end
 
   def cohorts
+    load
+  end
+
+  protected
+
+  def load
+    unless loaded?
+      @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query }
+      @loaded = true
+    end
+
+    @values
+  end
+
+  def perform_query
     sql = <<-SQL.squish
       SELECT axis.*, (
         WITH new_users AS (
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 0713aa471..2c16689bd 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -499,7 +499,7 @@ class FeedManager
 
     return false if active_filters.empty?
 
-    combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) }
+    combined_regex = Regexp.union(active_filters)
     status         = status.reblog if status.reblog?
 
     combined_text = [
@@ -549,7 +549,7 @@ class FeedManager
       end
     else
       # A reblog may reach earlier than the original status because of the
-      # delay of the worker deliverying the original status, the late addition
+      # delay of the worker delivering the original status, the late addition
       # by merging timelines, and other reasons.
       # If such a reblog already exists, just do not re-insert it into the feed.
       return false unless redis.zscore(reblog_key, status.id).nil?
diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb
index e07ebfffe..c685d7b6f 100644
--- a/app/lib/search_query_transformer.rb
+++ b/app/lib/search_query_transformer.rb
@@ -2,19 +2,21 @@
 
 class SearchQueryTransformer < Parslet::Transform
   class Query
-    attr_reader :should_clauses, :must_not_clauses, :must_clauses
+    attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses
 
     def initialize(clauses)
       grouped = clauses.chunk(&:operator).to_h
       @should_clauses = grouped.fetch(:should, [])
       @must_not_clauses = grouped.fetch(:must_not, [])
       @must_clauses = grouped.fetch(:must, [])
+      @filter_clauses = grouped.fetch(:filter, [])
     end
 
     def apply(search)
       should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
       must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
       must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
+      filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) }
       search.query.minimum_should_match(1)
     end
 
@@ -30,6 +32,15 @@ class SearchQueryTransformer < Parslet::Transform
         raise "Unexpected clause type: #{clause}"
       end
     end
+
+    def clause_to_filter(clause)
+      case clause
+      when PrefixClause
+        { term: { clause.filter => clause.term } }
+      else
+        raise "Unexpected clause type: #{clause}"
+      end
+    end
   end
 
   class Operator
@@ -69,11 +80,33 @@ class SearchQueryTransformer < Parslet::Transform
     end
   end
 
+  class PrefixClause
+    attr_reader :filter, :operator, :term
+
+    def initialize(prefix, term)
+      @operator = :filter
+      case prefix
+      when 'from'
+        @filter = :account_id
+        username, domain = term.split('@')
+        account = Account.find_remote(username, domain)
+
+        raise "Account not found: #{term}" unless account
+
+        @term = account.id
+      else
+        raise "Unknown prefix: #{prefix}"
+      end
+    end
+  end
+
   rule(clause: subtree(:clause)) do
     prefix   = clause[:prefix][:term].to_s if clause[:prefix]
     operator = clause[:operator]&.to_s
 
-    if clause[:term]
+    if clause[:prefix]
+      PrefixClause.new(prefix, clause[:term].to_s)
+    elsif clause[:term]
       TermClause.new(prefix, operator, clause[:term].to_s)
     elsif clause[:shortcode]
       TermClause.new(prefix, operator, ":#{clause[:term]}:")
diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb
index 03e40f923..2896620cb 100644
--- a/app/lib/video_metadata_extractor.rb
+++ b/app/lib/video_metadata_extractor.rb
@@ -2,7 +2,7 @@
 
 class VideoMetadataExtractor
   attr_reader :duration, :bitrate, :video_codec, :audio_codec,
-              :colorspace, :width, :height, :frame_rate
+              :colorspace, :width, :height, :frame_rate, :r_frame_rate
 
   def initialize(path)
     @path     = path
@@ -42,6 +42,7 @@ class VideoMetadataExtractor
         @width       = video_stream[:width]
         @height      = video_stream[:height]
         @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
+        @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
       end
 
       if (audio_stream = audio_streams.first)