From cdb101340a20183a82889f811d9311c370c855e5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Jun 2018 15:34:36 +0200 Subject: Keyword/phrase filtering (#7905) * Add keyword filtering GET|POST /api/v1/filters GET|PUT|DELETE /api/v1/filters/:id - Irreversible filters can drop toots from home or notifications - Other filters can hide toots through the client app - Filters use a phrase valid in particular contexts, expiration * Make sure expired filters don't get applied client-side * Add missing API methods * Remove "regex filter" from column settings * Add tests * Add test for FeedManager * Add CustomFilter test * Add UI for managing filters * Add streaming API event to allow syncing filters * Fix tests --- db/migrate/20180628181026_create_custom_filters.rb | 13 +++++++++++++ db/schema.rb | 14 +++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20180628181026_create_custom_filters.rb (limited to 'db') diff --git a/db/migrate/20180628181026_create_custom_filters.rb b/db/migrate/20180628181026_create_custom_filters.rb new file mode 100644 index 000000000..d19cf2e9d --- /dev/null +++ b/db/migrate/20180628181026_create_custom_filters.rb @@ -0,0 +1,13 @@ +class CreateCustomFilters < ActiveRecord::Migration[5.2] + def change + create_table :custom_filters do |t| + t.belongs_to :account, foreign_key: { on_delete: :cascade } + t.datetime :expires_at + t.text :phrase, null: false, default: '' + t.string :context, array: true, null: false, default: [] + t.boolean :irreversible, null: false, default: false + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 2853aef94..661fc8179 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_06_17_162849) do +ActiveRecord::Schema.define(version: 2018_06_28_181026) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -143,6 +143,17 @@ ActiveRecord::Schema.define(version: 2018_06_17_162849) do t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true end + create_table "custom_filters", force: :cascade do |t| + t.bigint "account_id" + t.datetime "expires_at" + t.text "phrase", default: "", null: false + t.string "context", default: [], null: false, array: true + t.boolean "irreversible", default: false, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_custom_filters_on_account_id" + end + create_table "domain_blocks", force: :cascade do |t| t.string "domain", default: "", null: false t.datetime "created_at", null: false @@ -561,6 +572,7 @@ ActiveRecord::Schema.define(version: 2018_06_17_162849) do add_foreign_key "blocks", "accounts", name: "fk_4269e03e65", on_delete: :cascade add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade + add_foreign_key "custom_filters", "accounts", on_delete: :cascade add_foreign_key "favourites", "accounts", name: "fk_5eb6c2b873", on_delete: :cascade add_foreign_key "favourites", "statuses", name: "fk_b0e856845e", on_delete: :cascade add_foreign_key "follow_requests", "accounts", column: "target_account_id", name: "fk_9291ec025d", on_delete: :cascade -- cgit From 1ca4e51eb38de6de81cedf3ddcdaa626f1d1c569 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 9 Jul 2018 02:22:09 +0200 Subject: Add option to not consider word boundaries when processing keyword filtering (#7975) * Add option to not consider word boundaries when filtering phrases * Add a few tests for keyword/phrase filtering --- app/controllers/api/v1/filters_controller.rb | 2 +- app/javascript/mastodon/selectors/index.js | 5 +++- app/lib/feed_manager.rb | 11 ++++++++- app/models/custom_filter.rb | 1 + app/serializers/rest/filter_serializer.rb | 2 +- app/views/filters/_fields.html.haml | 3 +++ ...180707154237_add_whole_word_to_custom_filter.rb | 17 +++++++++++++ db/schema.rb | 3 ++- spec/lib/feed_manager_spec.rb | 28 +++++++++++++++++----- 9 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 db/migrate/20180707154237_add_whole_word_to_custom_filter.rb (limited to 'db') diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb index a98080d1d..e5ebaff4d 100644 --- a/app/controllers/api/v1/filters_controller.rb +++ b/app/controllers/api/v1/filters_controller.rb @@ -43,6 +43,6 @@ class Api::V1::FiltersController < Api::BaseController end def resource_params - params.permit(:phrase, :expires_in, :irreversible, context: []) + params.permit(:phrase, :expires_in, :irreversible, :whole_word, context: []) end end diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index 7aa7569a0..d0212c379 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -45,7 +45,10 @@ export const regexFromFilters = filters => { return null; } - return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).map(expr => `\\b${expr}\\b`).join('|'), 'i'); + return new RegExp(filters.map(filter => { + let expr = escapeRegExp(filter.get('phrase')); + return filter.get('whole_word') ? `\\b${expr}\\b` : expr; + }).join('|'), 'i'); }; export const makeGetStatus = () => { diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 55c72d0ea..c247ab21d 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -200,7 +200,16 @@ class FeedManager active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - active_filters.map! { |filter| Regexp.new("\\b#{Regexp.escape(filter.phrase)}\\b", true) } + active_filters.map! do |filter| + if filter.whole_word + sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : '' + eb = filter.phrase =~ /[[:word:]]\Z/ ? '\b' : '' + + /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ + else + /#{Regexp.escape(filter.phrase)}/i + end + end return false if active_filters.empty? diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 2c1a54375..342207a55 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -8,6 +8,7 @@ # expires_at :datetime # phrase :text default(""), not null # context :string default([]), not null, is an Array +# whole_word :boolean default(TRUE), not null # irreversible :boolean default(FALSE), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/serializers/rest/filter_serializer.rb b/app/serializers/rest/filter_serializer.rb index 51340aa79..3134be371 100644 --- a/app/serializers/rest/filter_serializer.rb +++ b/app/serializers/rest/filter_serializer.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class REST::FilterSerializer < ActiveModel::Serializer - attributes :id, :phrase, :context, :expires_at, + attributes :id, :phrase, :context, :whole_word, :expires_at, :irreversible end diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml index af5d648b8..a5a3f0337 100644 --- a/app/views/filters/_fields.html.haml +++ b/app/views/filters/_fields.html.haml @@ -7,5 +7,8 @@ .fields-group = f.input :irreversible, wrapper: :with_label +.fields-group + = f.input :whole_word, wrapper: :with_label + .fields-group = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, prompt: I18n.t('invites.expires_in_prompt') diff --git a/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb b/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb new file mode 100644 index 000000000..63ecb8741 --- /dev/null +++ b/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb @@ -0,0 +1,17 @@ +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class AddWholeWordToCustomFilter < ActiveRecord::Migration[5.2] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def change + safety_assured do + add_column_with_default :custom_filters, :whole_word, :boolean, default: true, allow_null: false + end + end + + def down + remove_column :custom_filters, :whole_word + end +end diff --git a/db/schema.rb b/db/schema.rb index 661fc8179..02032c548 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_06_28_181026) do +ActiveRecord::Schema.define(version: 2018_07_07_154237) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -149,6 +149,7 @@ ActiveRecord::Schema.define(version: 2018_06_28_181026) do t.text "phrase", default: "", null: false t.string "context", default: [], null: false, array: true t.boolean "irreversible", default: false, null: false + t.boolean "whole_word", default: true, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_custom_filters_on_account_id" diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index d1b847675..7535e144d 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -127,12 +127,28 @@ RSpec.describe FeedManager do expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true end - it 'returns true if status contains irreversibly muted phrase' do - alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) - alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + context 'for irreversibly muted phrases' do + it 'considers word boundaries when matching' do + alice.custom_filters.create!(phrase: 'bob', context: %w(home), irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'bobcats', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be_falsy + end + + it 'returns true if phrase is contained' do + alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) + alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + end + + it 'matches substrings if whole_word is false' do + alice.custom_filters.create!(phrase: 'take', context: %w(home), whole_word: false, irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'shiitake', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + end end end -- cgit