Remove spam check and dependency on nilsimsa gem (#16011)

8 files changed, 0 insertions, 221 deletions

diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb
index 4422825ee..c829ed98f 100644
--- a/app/controllers/admin/dashboard_controller.rb
+++ b/app/controllers/admin/dashboard_controller.rb
@@ -35,7 +35,6 @@ module Admin
       @whitelist_enabled     = whitelist_mode?
       @profile_directory     = Setting.profile_directory
       @timeline_preview      = Setting.timeline_preview
-      @spam_check_enabled    = Setting.spam_check_enabled
       @trends_enabled        = Setting.trends
     end
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 0fa306cdd..9f6dd9ce0 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -88,7 +88,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
-    check_for_spam
     distribute(@status)
     forward_for_reply
   end
@@ -492,10 +491,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Tombstone.exists?(uri: object_uri)
   end
 
-  def check_for_spam
-    SpamCheck.perform(@status)
-  end
-
   def forward_for_reply
     return unless @status.distributable? && @json['signature'].present? && reply_to_local?
 
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
deleted file mode 100644
index dcb2db9ca..000000000
--- a/app/lib/spam_check.rb
+++ /dev/null
@@ -1,198 +0,0 @@
-# frozen_string_literal: true
-
-class SpamCheck
-  include Redisable
-  include ActionView::Helpers::TextHelper
-
-  # Threshold over which two Nilsimsa values are considered
-  # to refer to the same text
-  NILSIMSA_COMPARE_THRESHOLD = 95
-
-  # Nilsimsa doesn't work well on small inputs, so below
-  # this size, we check only for exact matches with MD5
-  NILSIMSA_MIN_SIZE = 10
-
-  # How long to keep the trail of digests between updates,
-  # there is no reason to store it forever
-  EXPIRE_SET_AFTER = 1.week.seconds
-
-  # How many digests to keep in an account's trail. If it's
-  # too small, spam could rotate around different message templates
-  MAX_TRAIL_SIZE = 10
-
-  # How many detected duplicates to allow through before
-  # considering the message as spam
-  THRESHOLD = 5
-
-  def initialize(status)
-    @account = status.account
-    @status  = status
-  end
-
-  def skip?
-    disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
-  end
-
-  def spam?
-    if insufficient_data?
-      false
-    elsif nilsimsa?
-      digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
-    else
-      digests_over_threshold?('md5') { |_, other_digest| other_digest == digest }
-    end
-  end
-
-  def flag!
-    auto_report_status!
-  end
-
-  def remember!
-    # The scores in sorted sets don't actually have enough bits to hold an exact
-    # value of our snowflake IDs, so we use it only for its ordering property. To
-    # get the correct status ID back, we have to save it in the string value
-
-    redis.zadd(redis_key, @status.id, digest_with_algorithm)
-    redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1))
-    redis.expire(redis_key, EXPIRE_SET_AFTER)
-  end
-
-  def reset!
-    redis.del(redis_key)
-  end
-
-  def hashable_text
-    return @hashable_text if defined?(@hashable_text)
-
-    @hashable_text = @status.text
-    @hashable_text = remove_mentions(@hashable_text)
-    @hashable_text = strip_tags(@hashable_text) unless @status.local?
-    @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
-    @hashable_text = remove_whitespace(@hashable_text)
-  end
-
-  def insufficient_data?
-    hashable_text.blank?
-  end
-
-  def digest
-    @digest ||= begin
-      if nilsimsa?
-        Nilsimsa.new(hashable_text).hexdigest
-      else
-        Digest::MD5.hexdigest(hashable_text)
-      end
-    end
-  end
-
-  def digest_with_algorithm
-    if nilsimsa?
-      ['nilsimsa', digest, @status.id].join(':')
-    else
-      ['md5', digest, @status.id].join(':')
-    end
-  end
-
-  class << self
-    def perform(status)
-      spam_check = new(status)
-
-      return if spam_check.skip?
-
-      if spam_check.spam?
-        spam_check.flag!
-      else
-        spam_check.remember!
-      end
-    end
-  end
-
-  private
-
-  def disabled?
-    !Setting.spam_check_enabled
-  end
-
-  def remove_mentions(text)
-    return text.gsub(Account::MENTION_RE, '') if @status.local?
-
-    Nokogiri::HTML.fragment(text).tap do |html|
-      mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
-
-      html.traverse do |element|
-        element.unlink if element.name == 'a' && mentions.include?(element['href'])
-      end
-    end.to_s
-  end
-
-  def normalize_unicode(text)
-    text.unicode_normalize(:nfkc).downcase
-  end
-
-  def remove_whitespace(text)
-    text.gsub(/\s+/, ' ').strip
-  end
-
-  def auto_report_status!
-    status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
-    ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected'))
-  end
-
-  def already_flagged?
-    @account.silenced? || @account.targeted_reports.unresolved.where(account_id: -99).exists?
-  end
-
-  def trusted?
-    @account.trust_level > Account::TRUST_LEVELS[:untrusted] || (@account.local? && @account.user_staff?)
-  end
-
-  def no_unsolicited_mentions?
-    @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
-  end
-
-  def solicited_reply?
-    !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
-  end
-
-  def nilsimsa_compare_value(first, second)
-    first  = [first].pack('H*')
-    second = [second].pack('H*')
-    bits   = 0
-
-    0.upto(31) do |i|
-      bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
-    end
-
-    128 - bits # -128 <= Nilsimsa Compare Value <= 128
-  end
-
-  def nilsimsa?
-    hashable_text.size > NILSIMSA_MIN_SIZE
-  end
-
-  def other_digests
-    redis.zrange(redis_key, 0, -1)
-  end
-
-  def digests_over_threshold?(filter_algorithm)
-    other_digests.select do |record|
-      algorithm, other_digest, status_id = record.split(':')
-
-      next unless algorithm == filter_algorithm
-
-      yield algorithm, other_digest, status_id
-    end.size >= THRESHOLD
-  end
-
-  def matching_status_ids
-    if nilsimsa?
-      other_digests.filter_map { |record| record.split(':')[2] if record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }
-    else
-      other_digests.filter_map { |record| record.split(':')[2] if record.start_with?('md5') && record.split(':')[1] == digest }
-    end
-  end
-
-  def redis_key
-    @redis_key ||= "spam_check:#{@account.id}"
-  end
-end
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index e9f78da21..b5c3dcdbe 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -29,7 +29,6 @@ class Form::AdminSettings
     thumbnail
     hero
     mascot
-    spam_check_enabled
     trends
     trendable_by_default
     show_domain_blocks
@@ -48,7 +47,6 @@ class Form::AdminSettings
     show_known_fediverse_at_about_page
     preview_sensitive_media
     profile_directory
-    spam_check_enabled
     trends
     trendable_by_default
     noindex
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 4c02c7865..73dbb1834 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -43,7 +43,6 @@ class ProcessMentionsService < BaseService
     end
 
     status.save!
-    check_for_spam(status)
 
     mentions.each { |mention| create_notification(mention) }
   end
@@ -72,8 +71,4 @@ class ProcessMentionsService < BaseService
   def resolve_account_service
     ResolveAccountService.new
   end
-
-  def check_for_spam(status)
-    SpamCheck.perform(status)
-  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index d6043fb5d..d642beeaa 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -41,7 +41,6 @@ class RemoveStatusService < BaseService
           remove_from_hashtags
           remove_from_public
           remove_from_media if @status.media_attachments.any?
-          remove_from_spam_check
           remove_media
         end
 
@@ -163,10 +162,6 @@ class RemoveStatusService < BaseService
     @status.media_attachments.destroy_all
   end
 
-  def remove_from_spam_check
-    redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
-  end
-
   def lock_options
     { redis: Redis.current, key: "distribute:#{@status.id}" }
   end
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index 205538402..57a753e6b 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -77,8 +77,6 @@
           = feature_hint(link_to(t('admin.dashboard.trends'), edit_admin_settings_path), @trends_enabled)
         %li
           = feature_hint(link_to(t('admin.dashboard.feature_relay'), admin_relays_path), @relay_enabled)
-        %li
-          = feature_hint(link_to(t('admin.dashboard.feature_spam_check'), edit_admin_settings_path), @spam_check_enabled)
 
   .dashboard__widgets__versions
     %div
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 159bd4b0a..0e705f205 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -92,9 +92,6 @@
     .fields-group
       = f.input :noindex, as: :boolean, wrapper: :with_label, label: t('admin.settings.default_noindex.title'), hint: t('admin.settings.default_noindex.desc_html')
 
-  .fields-group
-    = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
-
   %hr.spacer/
 
   .fields-group