Merge remote-tracking branch 'glitch/main'

10 files changed, 84 insertions, 45 deletions

diff --git a/app/lib/activitypub/parser/media_attachment_parser.rb b/app/lib/activitypub/parser/media_attachment_parser.rb
index 30bea1f0e..656be84b7 100644
--- a/app/lib/activitypub/parser/media_attachment_parser.rb
+++ b/app/lib/activitypub/parser/media_attachment_parser.rb
@@ -50,7 +50,7 @@ class ActivityPub::Parser::MediaAttachmentParser
     components = begin
       blurhash = @json['blurhash']
 
-      if blurhash.present? && /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
+      if blurhash.present? && /^[\w#$%*+,-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
         Blurhash.components(blurhash)
       end
     end
diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb
index 877a42ef6..f512635ab 100644
--- a/app/lib/admin/system_check.rb
+++ b/app/lib/admin/system_check.rb
@@ -8,11 +8,11 @@ class Admin::SystemCheck
     Admin::SystemCheck::ElasticsearchCheck,
   ].freeze
 
-  def self.perform
+  def self.perform(current_user)
     ACTIVE_CHECKS.each_with_object([]) do |klass, arr|
-      check = klass.new
+      check = klass.new(current_user)
 
-      if check.pass?
+      if check.skip? || check.pass?
         arr
       else
         arr << check.message
diff --git a/app/lib/admin/system_check/base_check.rb b/app/lib/admin/system_check/base_check.rb
index fcad8daca..c2974c218 100644
--- a/app/lib/admin/system_check/base_check.rb
+++ b/app/lib/admin/system_check/base_check.rb
@@ -1,6 +1,16 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::BaseCheck
+  attr_reader :current_user
+
+  def initialize(current_user)
+    @current_user = current_user
+  end
+
+  def skip?
+    false
+  end
+
   def pass?
     raise NotImplementedError
   end
diff --git a/app/lib/admin/system_check/database_schema_check.rb b/app/lib/admin/system_check/database_schema_check.rb
index b93d1954e..c2f01fd55 100644
--- a/app/lib/admin/system_check/database_schema_check.rb
+++ b/app/lib/admin/system_check/database_schema_check.rb
@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::DatabaseSchemaCheck < Admin::SystemCheck::BaseCheck
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
   def pass?
     !ActiveRecord::Base.connection.migration_context.needs_migration?
   end
diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb
index 1b48a5415..8aee18267 100644
--- a/app/lib/admin/system_check/elasticsearch_check.rb
+++ b/app/lib/admin/system_check/elasticsearch_check.rb
@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
   def pass?
     return true unless Chewy.enabled?
 
@@ -32,8 +36,4 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
   def compatible_version?
     Gem::Version.new(running_version) >= Gem::Version.new(required_version)
   end
-
-  def missing_queues
-    @missing_queues ||= Sidekiq::ProcessSet.new.reduce(SIDEKIQ_QUEUES) { |queues, process| queues - process['queues'] }
-  end
 end
diff --git a/app/lib/admin/system_check/rules_check.rb b/app/lib/admin/system_check/rules_check.rb
index 1fbdf955d..8206a5df3 100644
--- a/app/lib/admin/system_check/rules_check.rb
+++ b/app/lib/admin/system_check/rules_check.rb
@@ -3,6 +3,10 @@
 class Admin::SystemCheck::RulesCheck < Admin::SystemCheck::BaseCheck
   include RoutingHelper
 
+  def skip?
+    !current_user.can?(:manage_rules)
+  end
+
   def pass?
     Rule.kept.exists?
   end
diff --git a/app/lib/admin/system_check/sidekiq_process_check.rb b/app/lib/admin/system_check/sidekiq_process_check.rb
index 22446edaf..648811d6c 100644
--- a/app/lib/admin/system_check/sidekiq_process_check.rb
+++ b/app/lib/admin/system_check/sidekiq_process_check.rb
@@ -9,6 +9,10 @@ class Admin::SystemCheck::SidekiqProcessCheck < Admin::SystemCheck::BaseCheck
     scheduler
   ).freeze
 
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
   def pass?
     missing_queues.empty?
   end
diff --git a/app/lib/ascii_folding.rb b/app/lib/ascii_folding.rb
new file mode 100644
index 000000000..1798d3d0e
--- /dev/null
+++ b/app/lib/ascii_folding.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class ASCIIFolding
+  NON_ASCII_CHARS        = 'ÀÁÂÃÄÅàáâãäåĀāĂ㥹ÇçĆćĈĉĊċČčÐðĎďĐđÈÉÊËèéêëĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħÌÍÎÏìíîïĨĩĪīĬĭĮįİıĴĵĶķĸĹĺĻļĽľĿŀŁłÑñŃńŅņŇňʼnŊŋÒÓÔÕÖØòóôõöøŌōŎŏŐőŔŕŖŗŘřŚśŜŝŞşŠšſŢţŤťŦŧÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųŴŵÝýÿŶŷŸŹźŻżŽž'
+  EQUIVALENT_ASCII_CHARS = 'AAAAAAaaaaaaAaAaAaCcCcCcCcCcDdDdDdEEEEeeeeEeEeEeEeEeGgGgGgGgHhHhIIIIiiiiIiIiIiIiIiJjKkkLlLlLlLlLlNnNnNnNnnNnOOOOOOooooooOoOoOoRrRrRrSsSsSsSssTtTtTtUUUUuuuuUuUuUuUuUuUuWwYyyYyYZzZzZz'
+
+  def fold(str)
+    str.tr(NON_ASCII_CHARS, EQUIVALENT_ASCII_CHARS)
+  end
+end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 235d6fedd..c607223fc 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -47,6 +47,8 @@ class FeedManager
       filter_from_mentions?(status, receiver.id)
     when :direct
       filter_from_direct?(status, receiver.id)
+    when :tags
+      filter_from_tags?(status, receiver.id, build_crutches(receiver.id, [status]))
     else
       false
     end
@@ -58,7 +60,7 @@ class FeedManager
   # @param [Boolean] update
   # @return [Boolean]
   def push_to_home(account, status, update: false)
-    return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+    return false unless add_to_feed(:home, account.id, status, aggregate_reblogs: account.user&.aggregates_reblogs?)
 
     trim(:home, account.id)
     PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}", { 'update' => update }) if push_update_required?("timeline:#{account.id}")
@@ -71,7 +73,7 @@ class FeedManager
   # @param [Boolean] update
   # @return [Boolean]
   def unpush_from_home(account, status, update: false)
-    return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+    return false unless remove_from_feed(:home, account.id, status, aggregate_reblogs: account.user&.aggregates_reblogs?)
 
     redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update
     true
@@ -83,7 +85,7 @@ class FeedManager
   # @param [Boolean] update
   # @return [Boolean]
   def push_to_list(list, status, update: false)
-    return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+    return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?)
 
     trim(:list, list.id)
     PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", { 'update' => update }) if push_update_required?("timeline:list:#{list.id}")
@@ -96,7 +98,7 @@ class FeedManager
   # @param [Boolean] update
   # @return [Boolean]
   def unpush_from_list(list, status, update: false)
-    return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+    return false unless remove_from_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?)
 
     redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update
     true
@@ -145,7 +147,7 @@ class FeedManager
     statuses.each do |status|
       next if filter_from_home?(status, into_account.id, crutches)
 
-      add_to_feed(:home, into_account.id, status, aggregate)
+      add_to_feed(:home, into_account.id, status, aggregate_reblogs: aggregate)
     end
 
     trim(:home, into_account.id)
@@ -171,7 +173,7 @@ class FeedManager
     statuses.each do |status|
       next if filter_from_home?(status, list.account_id, crutches) || filter_from_list?(status, list)
 
-      add_to_feed(:list, list.id, status, aggregate)
+      add_to_feed(:list, list.id, status, aggregate_reblogs: aggregate)
     end
 
     trim(:list, list.id)
@@ -186,7 +188,7 @@ class FeedManager
     timeline_status_ids = redis.zrange(timeline_key, 0, -1)
 
     from_account.statuses.select('id, reblog_of_id').where(id: timeline_status_ids).reorder(nil).find_each do |status|
-      remove_from_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?)
+      remove_from_feed(:home, into_account.id, status, aggregate_reblogs: into_account.user&.aggregates_reblogs?)
     end
   end
 
@@ -199,7 +201,7 @@ class FeedManager
     timeline_status_ids = redis.zrange(timeline_key, 0, -1)
 
     from_account.statuses.select('id, reblog_of_id').where(id: timeline_status_ids).reorder(nil).find_each do |status|
-      remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+      remove_from_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?)
     end
   end
 
@@ -262,7 +264,7 @@ class FeedManager
     timeline_key = key(:home, account.id)
 
     account.statuses.limit(limit).each do |status|
-      add_to_feed(:home, account.id, status, aggregate)
+      add_to_feed(:home, account.id, status, aggregate_reblogs: aggregate)
     end
 
     account.following.includes(:account_stat).find_each do |target_account|
@@ -282,7 +284,7 @@ class FeedManager
       statuses.each do |status|
         next if filter_from_home?(status, account.id, crutches)
 
-        add_to_feed(:home, account.id, status, aggregate)
+        add_to_feed(:home, account.id, status, aggregate_reblogs: aggregate)
       end
 
       trim(:home, account.id)
@@ -401,7 +403,6 @@ class FeedManager
   def filter_from_home?(status, receiver_id, crutches)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
-    return true  if phrase_filtered?(status, receiver_id, :home)
 
     check_for_blocks = crutches[:active_mentions][status.id] || []
     check_for_blocks.concat([status.account_id])
@@ -437,7 +438,6 @@ class FeedManager
   # @return [Boolean]
   def filter_from_mentions?(status, receiver_id)
     return true if receiver_id == status.account_id
-    return true if phrase_filtered?(status, receiver_id, :notifications)
 
     # This filter is called from NotifyService, but already after the sender of
     # the notification has been checked for mute/block. Therefore, it's not
@@ -476,32 +476,14 @@ class FeedManager
     false
   end
 
-  # Check if the status hits a phrase filter
+  # Check if a status should not be added to the home feed when it comes
+  # from a followed hashtag
   # @param [Status] status
   # @param [Integer] receiver_id
-  # @param [Symbol] context
+  # @param [Hash] crutches
   # @return [Boolean]
-  def phrase_filtered?(status, receiver_id, context)
-    active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a
-
-    active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? }
-
-    active_filters.map! do |filter|
-      if filter.whole_word
-        sb = /\A[[:word:]]/.match?(filter.phrase) ? '\b' : ''
-        eb = /[[:word:]]\z/.match?(filter.phrase) ? '\b' : ''
-
-        /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/
-      else
-        /#{Regexp.escape(filter.phrase)}/i
-      end
-    end
-
-    return false if active_filters.empty?
-
-    combined_regex = Regexp.union(active_filters)
-
-    combined_regex.match?(status.proper.searchable_text)
+  def filter_from_tags?(status, receiver_id, crutches)
+    receiver_id != status.account_id && (((crutches[:active_mentions][status.id] || []) + [status.account_id]).any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } || crutches[:blocked_by][status.account_id] || crutches[:domain_blocking][status.account.domain])
   end
 
   # Adds a status to an account's feed, returning true if a status was
@@ -513,7 +495,7 @@ class FeedManager
   # @param [Status] status
   # @param [Boolean] aggregate_reblogs
   # @return [Boolean]
-  def add_to_feed(timeline_type, account_id, status, aggregate_reblogs = true)
+  def add_to_feed(timeline_type, account_id, status, aggregate_reblogs: true)
     timeline_key = key(timeline_type, account_id)
     reblog_key   = key(timeline_type, account_id, 'reblogs')
 
@@ -561,7 +543,7 @@ class FeedManager
   # @param [Status] status
   # @param [Boolean] aggregate_reblogs
   # @return [Boolean]
-  def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs = true)
+  def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs: true)
     timeline_key = key(timeline_type, account_id)
     reblog_key   = key(timeline_type, account_id, 'reblogs')
 
diff --git a/app/lib/hashtag_normalizer.rb b/app/lib/hashtag_normalizer.rb
new file mode 100644
index 000000000..c1f99e163
--- /dev/null
+++ b/app/lib/hashtag_normalizer.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class HashtagNormalizer
+  def normalize(str)
+    remove_invalid_characters(ascii_folding(lowercase(cjk_width(str))))
+  end
+
+  private
+
+  def remove_invalid_characters(str)
+    str.gsub(/[^[:alnum:]#{Tag::HASHTAG_SEPARATORS}]/, '')
+  end
+
+  def ascii_folding(str)
+    ASCIIFolding.new.fold(str)
+  end
+
+  def lowercase(str)
+    str.mb_chars.downcase.to_s
+  end
+
+  def cjk_width(str)
+    str.unicode_normalize(:nfkc)
+  end
+end