filters now have options to separately match post text, content warnings/titles, & hashtags + option to filter threads containing a matching post

14 files changed, 122 insertions, 50 deletions

diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb
index 496964394..48177f41a 100644
--- a/app/controllers/api/v1/filters_controller.rb
+++ b/app/controllers/api/v1/filters_controller.rb
@@ -43,6 +43,6 @@ class Api::V1::FiltersController < Api::BaseController
   end
 
   def resource_params
-    params.permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :media_only, context: [])
+    params.permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :media_only, :status_text, :spoiler, :tags, context: [])
   end
 end
diff --git a/app/controllers/filters_controller.rb b/app/controllers/filters_controller.rb
index 2f6f98272..55ee05833 100644
--- a/app/controllers/filters_controller.rb
+++ b/app/controllers/filters_controller.rb
@@ -58,7 +58,7 @@ class FiltersController < ApplicationController
   end
 
   def resource_params
-    params.require(:custom_filter).permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :media_only, context: [])
+    params.require(:custom_filter).permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :spoiler, :tags, :thread, :media_only, context: [])
   end
 
   def set_body_classes
diff --git a/app/helpers/filter_helper.rb b/app/helpers/filter_helper.rb
new file mode 100644
index 000000000..861f37551
--- /dev/null
+++ b/app/helpers/filter_helper.rb
@@ -0,0 +1,51 @@
+module FilterHelper
+	def phrase_filtered?(status, receiver_id, context)
+    filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a
+
+    filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? }
+
+    if status.media_attachments.any?
+      filters.delete_if { |filter| filter.exclude_media }
+    else
+      filters.delete_if { |filter| filter.media_only }
+    end
+
+    return false if filters.empty?
+
+    status = status.reblog if status.reblog?
+    status_text = Formatter.instance.plaintext(status)
+    spoiler_text = status.spoiler_text
+    tags = status.tags.pluck(:name).join("\n")
+
+    filters.each do |filter|
+      if filter.whole_word
+        sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : ''
+        eb = filter.phrase =~ /[[:word:]]\z/ ? '\b' : ''
+
+        regex = /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/
+      else
+        regex = /#{Regexp.escape(filter.phrase)}/i
+      end
+
+      matched = false
+      matched = true unless regex.match(status_text).nil?
+      matched = true unless spoiler_text.blank? || regex.match(spoiler_text).nil?
+      matched = true unless tags.empty? || tags_regex.match(tags).nil?
+
+      if matched
+        filter_thread(receiver_id, status.conversation_id) if filter.thread
+        return true
+      end
+    end
+
+    false
+  end
+
+  def filter_thread(account_id, conversation_id)
+    Redis.cache.sadd("filtered_threads:#{account_id}", conversation_id)
+  end
+
+  def filtering_thread?(account_id, conversation_id)
+    Redis.cache.sismember("filtered_threads:#{account_id}", conversation_id)
+  end
+end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 9996291c2..6d8f46e6f 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -5,6 +5,7 @@ require 'singleton'
 class FeedManager
   include Singleton
   include Redisable
+  include FilterHelper
 
   MAX_ITEMS = 1666
 
@@ -156,6 +157,7 @@ class FeedManager
   def filter_from_home?(status, receiver_id)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
+    return true  if filtering_thread?(receiver_id, status.conversation_id)
     return true  if phrase_filtered?(status, receiver_id, :home)
 
     check_for_blocks = status.active_mentions.pluck(:account_id)
@@ -168,12 +170,12 @@ class FeedManager
 
     return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home)
 
-    if status.reply? && !status.in_reply_to_account_id.nil?                                                                      # Filter out if it's a reply
-      should_filter   = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists?         # and I'm not following the person it's a reply to
-      should_filter &&= receiver_id != status.in_reply_to_account_id                                                             # and it's not a reply to me
-      should_filter &&= status.account_id != status.in_reply_to_account_id                                                       # and it's not a self-reply
+    if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply
+      should_filter   = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists? # and I'm not following the person it's a reply to
+      should_filter &&= receiver_id != status.in_reply_to_account_id # and it's not a reply to me
+      should_filter &&= status.account_id != status.in_reply_to_account_id # and it's not a self-reply
       return should_filter
-    elsif status.reblog?                                                                                                         # Filter out a reblog
+    elsif status.reblog? # Filter out a reblog
       should_filter   = Follow.where(account_id: receiver_id, target_account_id: status.account_id, show_reblogs: false).exists? # if the reblogger's reblogs are suppressed
       should_filter ||= (status.reblog.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.reblog.account_id).exists?) # or if the account is silenced and I'm not following them
       should_filter ||= Block.where(account_id: status.reblog.account_id, target_account_id: receiver_id).exists?                # or if the author of the reblogged status is blocking me
@@ -186,6 +188,7 @@ class FeedManager
 
   def filter_from_mentions?(status, receiver_id)
     return true if receiver_id == status.account_id
+    return true if filtering_thread?(receiver_id, status.conversation_id)
     return true if phrase_filtered?(status, receiver_id, :notifications)
 
     # This filter is called from NotifyService, but already after the sender of
@@ -200,37 +203,6 @@ class FeedManager
     should_filter
   end
 
-  def phrase_filtered?(status, receiver_id, context)
-    active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a
-
-    active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? }
-
-    if status.media_attachments.any?
-      active_filters.delete_if { |filter| filter.exclude_media }
-    else
-      active_filters.delete_if { |filter| filter.media_only }
-    end
-
-    active_filters.map! do |filter|
-      if filter.whole_word
-        sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : ''
-        eb = filter.phrase =~ /[[:word:]]\z/ ? '\b' : ''
-
-        /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/
-      else
-        /#{Regexp.escape(filter.phrase)}/i
-      end
-    end
-
-    return false if active_filters.empty?
-
-    combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) }
-    status         = status.reblog if status.reblog?
-
-    !combined_regex.match(Formatter.instance.plaintext(status)).nil? ||
-      (status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?)
-  end
-
   # Adds a status to an account's feed, returning true if a status was
   # added, and false if it was not added to the feed. Note that this is
   # an internal helper: callers must call trim or push updates if
diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb
index f12986c93..a28c8282a 100644
--- a/app/lib/status_filter.rb
+++ b/app/lib/status_filter.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class StatusFilter
+  include FilterHelper
+
   attr_reader :status, :account
 
   def initialize(status, account, preloaded_relations = {})
@@ -21,6 +23,7 @@ class StatusFilter
   end
 
   def filtered_status?
+    return true if filtering_thread?(account.id, status.conversation_id)
     blocking_account? || blocking_domain? || muting_account? || filtered_reference?
   end
 
@@ -30,6 +33,9 @@ class StatusFilter
     return true if account.user_hides_replies_of_muted? && reply_to_muted?
     return true if account.user_hides_replies_of_blocker? && reply_to_blocker?
 
+    # filtered by user?
+    return true if phrase_filtered?(status, account.id, 'thread')
+
     # kajiht has no filters if status has no mentions
     return false if status&.mentions.blank?
 
@@ -43,23 +49,23 @@ class StatusFilter
     return true if account.user_hides_mentions_of_blocked? && mentioned_accounts.where.not(suspended_at: nil).exists?
 
     return true if mentioned_account_ids.any? do |mentioned_account_id|
-      return true if @preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && @preloaded_relations[:muting][mentioned_account_id]
-      return true if @preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && @preloaded_relations[:blocking][mentioned_account_id]
+      break true if @preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && @preloaded_relations[:muting][mentioned_account_id]
+      break true if @preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && @preloaded_relations[:blocking][mentioned_account_id]
 
       if @preloaded_relations[:blocked_by]
-        return true if account.user_hides_mentions_of_blocker? && @preloaded_relations[:blocked_by][mentioned_account_id]
+        break true if account.user_hides_mentions_of_blocker? && @preloaded_relations[:blocked_by][mentioned_account_id]
       else
-        return true if account.user_hides_mentions_of_blocker? && Block.where(account_id: mentioned_account_id, target_account_id: account.id).exists?
+        break true if account.user_hides_mentions_of_blocker? && Block.where(account_id: mentioned_account_id, target_account_id: account.id).exists?
       end
 
-      return false unless status.reply?
-      @preloaded_relations[:following] && account.user_hides_mentions_outside_scope? && status.private_visibility? && !@preloaded_relations[:following][mentioned_account_id]
+      break false unless status.reply? && status.private_visibility? && account.user_hides_mentions_outside_scope?
+      @preloaded_relations[:following] && !@preloaded_relations[:following][mentioned_account_id]
     end
 
     return true if !@preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && account.muting?(mentioned_account_ids)
     return true if !@preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && account.blocking?(mentioned_account_ids)
-    return false unless status.reply?
-    !@preloaded_relations[:following] && account.user_hides_mentions_outside_scope? && status.private_visibility? && (mentioned_account_ids - account.following_ids).any?
+    return false unless status.reply? && status.private_visibility? && account.user_hides_mentions_outside_scope?
+    !@preloaded_relations[:following] && (mentioned_account_ids - account.following_ids).any?
   end
 
   def reply_to_blocked?
diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb
index d4cb2206e..a1db3940c 100644
--- a/app/models/custom_filter.rb
+++ b/app/models/custom_filter.rb
@@ -14,6 +14,10 @@
 #  whole_word    :boolean          default(TRUE), not null
 #  exclude_media :boolean          default(FALSE), not null
 #  media_only    :boolean          default(FALSE), not null
+#  thread        :boolean          default(FALSE), not null
+#  spoiler       :boolean          default(FALSE), not null
+#  tags          :boolean          default(FALSE), not null
+#  status_text   :boolean          default(FALSE), not null
 #
 
 class CustomFilter < ApplicationRecord
@@ -45,6 +49,7 @@ class CustomFilter < ApplicationRecord
 
   def remove_cache
     Rails.cache.delete("filters:#{account_id}")
+    Rails.cache.delete("filtered_threads:#{account_id}")
     Redis.current.publish("timeline:#{account_id}", Oj.dump(event: :filters_changed))
   end
 
diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml
index 84b380f13..8229cb728 100644
--- a/app/views/filters/_fields.html.haml
+++ b/app/views/filters/_fields.html.haml
@@ -16,5 +16,9 @@
   = f.input :whole_word, wrapper: :with_label
 
 .fields-group
-  = f.input :exclude_media, wrapper: :with_label
+  = f.input :status_text, wrapper: :with_label
+  = f.input :spoiler, wrapper: :with_label
+  = f.input :tags, wrapper: :with_label
+  = f.input :thread, wrapper: :with_label
   = f.input :media_only, wrapper: :with_label
+  = f.input :exclude_media, wrapper: :with_label
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 82e4feabb..75ff9d69d 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -97,8 +97,12 @@ en:
         header: Header
         inbox_url: URL of the relay inbox
         irreversible: Drop instead of hide
-        exclude_media: Don't filter roars with attachments
-        media_only: Only filter roars with attachments
+        exclude_media: Filter roars WITHOUT attachments
+        status_text: Filter roars with matching body text
+        media_only: Filter roars with attachments
+        thread: Filter the entire thread this match is contained in
+        spoiler: Filter roars with matching content warnings or topics
+        tags: Filter roars with matching tag(s)
         locale: Interface language
         locked: Lock account
         max_uses: Max number of uses
diff --git a/db/migrate/20190801212756_add_thread_to_custom_filters.rb b/db/migrate/20190801212756_add_thread_to_custom_filters.rb
new file mode 100644
index 000000000..de2507451
--- /dev/null
+++ b/db/migrate/20190801212756_add_thread_to_custom_filters.rb
@@ -0,0 +1,5 @@
+class AddThreadToCustomFilters < ActiveRecord::Migration[5.2]
+  def change
+    safety_assured { add_column :custom_filters, :thread, :boolean, null: false, default: false }
+  end
+end
diff --git a/db/migrate/20190801213117_add_spoiler_to_custom_filters.rb b/db/migrate/20190801213117_add_spoiler_to_custom_filters.rb
new file mode 100644
index 000000000..63aba46ac
--- /dev/null
+++ b/db/migrate/20190801213117_add_spoiler_to_custom_filters.rb
@@ -0,0 +1,5 @@
+class AddSpoilerToCustomFilters < ActiveRecord::Migration[5.2]
+  def change
+    safety_assured { add_column :custom_filters, :spoiler, :boolean, null: false, default: false }
+  end
+end
diff --git a/db/migrate/20190801213606_add_tags_to_custom_filters.rb b/db/migrate/20190801213606_add_tags_to_custom_filters.rb
new file mode 100644
index 000000000..4ed1a6e34
--- /dev/null
+++ b/db/migrate/20190801213606_add_tags_to_custom_filters.rb
@@ -0,0 +1,5 @@
+class AddTagsToCustomFilters < ActiveRecord::Migration[5.2]
+  def change
+    safety_assured { add_column :custom_filters, :tags, :boolean, null: false, default: false }
+  end
+end
diff --git a/db/migrate/20190801222645_add_status_text_to_custom_filters.rb b/db/migrate/20190801222645_add_status_text_to_custom_filters.rb
new file mode 100644
index 000000000..cf4bbd28b
--- /dev/null
+++ b/db/migrate/20190801222645_add_status_text_to_custom_filters.rb
@@ -0,0 +1,5 @@
+class AddStatusTextToCustomFilters < ActiveRecord::Migration[5.2]
+  def change
+    safety_assured { add_column :custom_filters, :status_text, :boolean, null: false, default: false }
+  end
+end
diff --git a/db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb b/db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb
new file mode 100644
index 000000000..787c63929
--- /dev/null
+++ b/db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb
@@ -0,0 +1,6 @@
+class SetStatusTextBoolForExistingFilters < ActiveRecord::Migration[5.2]
+  def up
+    CustomFilter.where(status_text: false).in_batches.update_all(status_text: true)
+    CustomFilter.where(spoiler_text: false).in_batches.update_all(spoiler_text: true)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 4d7085515..9197404aa 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_30_213656) do
+ActiveRecord::Schema.define(version: 2019_08_01_222823) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -251,6 +251,10 @@ ActiveRecord::Schema.define(version: 2019_07_30_213656) do
     t.boolean "whole_word", default: true, null: false
     t.boolean "exclude_media", default: false, null: false
     t.boolean "media_only", default: false, null: false
+    t.boolean "thread", default: false, null: false
+    t.boolean "spoiler", default: false, null: false
+    t.boolean "tags", default: false, null: false
+    t.boolean "status_text", default: false, null: false
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end