From cdb101340a20183a82889f811d9311c370c855e5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Jun 2018 15:34:36 +0200 Subject: Keyword/phrase filtering (#7905) * Add keyword filtering GET|POST /api/v1/filters GET|PUT|DELETE /api/v1/filters/:id - Irreversible filters can drop toots from home or notifications - Other filters can hide toots through the client app - Filters use a phrase valid in particular contexts, expiration * Make sure expired filters don't get applied client-side * Add missing API methods * Remove "regex filter" from column settings * Add tests * Add test for FeedManager * Add CustomFilter test * Add UI for managing filters * Add streaming API event to allow syncing filters * Fix tests --- app/lib/feed_manager.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'app/lib') diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index c18c07b33..ee9185d34 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -153,6 +153,7 @@ class FeedManager def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) + return true if phrase_filtered?(status, receiver_id, :home) check_for_blocks = status.mentions.pluck(:account_id) check_for_blocks.concat([status.account_id]) @@ -177,6 +178,7 @@ class FeedManager def filter_from_mentions?(status, receiver_id) return true if receiver_id == status.account_id + return true if phrase_filtered?(status, receiver_id, :notifications) # This filter is called from NotifyService, but already after the sender of # the notification has been checked for mute/block. Therefore, it's not @@ -190,6 +192,20 @@ class FeedManager should_filter end + def phrase_filtered?(status, receiver_id, context) + active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a + + active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } + active_filters.map! { |filter| Regexp.new(Regexp.escape(filter.phrase), true) } + + return false if active_filters.empty? + + combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) } + + !combined_regex.match(status.text).nil? || + (status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?) + end + # Adds a status to an account's feed, returning true if a status was # added, and false if it was not added to the feed. Note that this is # an internal helper: callers must call trim or push updates if -- cgit From ddd0bb69e1d9ea98a0ea30717e9f25bcd50b41a3 Mon Sep 17 00:00:00 2001 From: MIYAGI Hikaru Date: Fri, 29 Jun 2018 22:36:02 +0900 Subject: Merge `HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY` into `ALLOW_ACCESS_TO_HIDDEN_SERVICE` (#7901) If Mastodon accesses to the hidden service via transparent proxy, it's needed to avoid checking whether it's a private address, since `.onion` is resolved to a private address. I was previously using the `HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY` to provide that function. However, I realized that using `HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY` is redundant, since this specification is always used with `ALLOW_ACCESS_TO_HIDDEN_SERVICE`. Therefore, I decided to integrate the setting of `HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY` into` ALLOW_ACCESS_TO_HIDDEN_SERVICE`. --- .env.production.sample | 2 -- app/lib/request.rb | 2 +- config/initializers/http_client_proxy.rb | 1 - docker-compose.yml | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) (limited to 'app/lib') diff --git a/.env.production.sample b/.env.production.sample index 3047f7595..ebb078878 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -229,5 +229,3 @@ STREAMING_CLUSTER_NUM=1 # http_proxy=http://gateway.local:8118 # Access control for hidden service. # ALLOW_ACCESS_TO_HIDDEN_SERVICE=true -# If you use transparent proxy to access to hidden service, uncomment following for skipping private address check. -# HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY=true diff --git a/app/lib/request.rb b/app/lib/request.rb index 397614fac..576ed23ca 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -154,7 +154,7 @@ class Request alias new open def thru_hidden_service?(host) - Rails.configuration.x.hidden_service_via_transparent_proxy && /\.(onion|i2p)$/.match(host) + Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(host) end end end diff --git a/config/initializers/http_client_proxy.rb b/config/initializers/http_client_proxy.rb index 52c595c5d..e607aff3c 100644 --- a/config/initializers/http_client_proxy.rb +++ b/config/initializers/http_client_proxy.rb @@ -11,7 +11,6 @@ Rails.application.configure do end config.x.access_to_hidden_service = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true' - config.x.hidden_service_via_transparent_proxy = ENV['HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY'] == 'true' end module Goldfinger diff --git a/docker-compose.yml b/docker-compose.yml index b31095a54..c54c73e44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,7 +83,6 @@ services: ## Uncomment to enable federation with tor instances along with adding the following ENV variables ## http_proxy=http://privoxy:8118 ## ALLOW_ACCESS_TO_HIDDEN_SERVICE=true -## HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY=true # tor: # build: https://github.com/usbsnowcrash/docker-tor.git # networks: -- cgit From da8fe8079e13758f45e5ba77cb8023c554ae193c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 3 Jul 2018 01:47:56 +0200 Subject: Re-add follow recommendations API (#7918) * Re-add follow recommendations API GET /api/v1/suggestions Removed in 8efa081f210d72ed450c39ac4cde0fd84fb3d3fb due to Neo4J dependency. The algorithm uses triadic closures, takes into account suspensions, blocks, mutes, domain blocks, excludes locked and moved accounts, and prefers more recently updated accounts. * Track interactions with people you don't follow Replying to, favouriting and reblogging someone you're not following will make them show up in follow recommendations. The interactions have different weights: - Replying is 1 - Favouriting is 10 (decidedly positive interaction, but private) - Reblogging is 20 Following them, muting or blocking will remove them from the list, obviously. * Remove triadic closures, ensure potential friendships are trimmed --- app/controllers/api/v1/suggestions_controller.rb | 21 +++++++ app/lib/potential_friendship_tracker.rb | 39 ++++++++++++ app/models/account.rb | 29 +-------- app/models/concerns/account_interactions.rb | 12 ++++ app/services/favourite_service.rb | 8 +++ app/services/post_status_service.rb | 7 +++ app/services/reblog_service.rb | 7 +++ config/routes.rb | 1 + .../api/v1/suggestions_controller_spec.rb | 35 +++++++++++ spec/models/account_spec.rb | 71 ---------------------- 10 files changed, 131 insertions(+), 99 deletions(-) create mode 100644 app/controllers/api/v1/suggestions_controller.rb create mode 100644 app/lib/potential_friendship_tracker.rb create mode 100644 spec/controllers/api/v1/suggestions_controller_spec.rb (limited to 'app/lib') diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb new file mode 100644 index 000000000..3abccedd5 --- /dev/null +++ b/app/controllers/api/v1/suggestions_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::SuggestionsController < Api::BaseController + include Authorization + + before_action -> { doorkeeper_authorize! :read } + before_action :require_user! + before_action :set_accounts + + respond_to :json + + def index + render json: @accounts, each_serializer: REST::AccountSerializer + end + + private + + def set_accounts + @accounts = PotentialFriendshipTracker.get(current_account.id, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT)) + end +end diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb new file mode 100644 index 000000000..362482669 --- /dev/null +++ b/app/lib/potential_friendship_tracker.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class PotentialFriendshipTracker + EXPIRE_AFTER = 90.days.seconds + MAX_ITEMS = 80 + + WEIGHTS = { + reply: 1, + favourite: 10, + reblog: 20, + }.freeze + + class << self + def record(account_id, target_account_id, action) + key = "interactions:#{account_id}" + weight = WEIGHTS[action] + + redis.zincrby(key, weight, target_account_id) + redis.zremrangebyrank(key, 0, -MAX_ITEMS) + redis.expire(key, EXPIRE_AFTER) + end + + def remove(account_id, target_account_id) + redis.zrem("interactions:#{account_id}", target_account_id) + end + + def get(account_id, limit: 20, offset: 0) + account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit) + return [] if account_ids.empty? + Account.searchable.where(id: account_ids) + end + + private + + def redis + Redis.current + end + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 40a45b1f8..1f720bf88 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -127,6 +127,7 @@ class Account < ApplicationRecord scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) } scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) } scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } + scope :searchable, -> { where(suspended: false).where(moved_to_account_id: nil) } delegate :email, :unconfirmed_email, @@ -309,34 +310,6 @@ class Account < ApplicationRecord DeliveryFailureTracker.filter(urls) end - def triadic_closures(account, limit: 5, offset: 0) - sql = <<-SQL.squish - WITH first_degree AS ( - SELECT target_account_id - FROM follows - WHERE account_id = :account_id - ) - SELECT accounts.* - FROM follows - INNER JOIN accounts ON follows.target_account_id = accounts.id - WHERE - account_id IN (SELECT * FROM first_degree) - AND target_account_id NOT IN (SELECT * FROM first_degree) - AND target_account_id NOT IN (:excluded_account_ids) - AND accounts.suspended = false - GROUP BY target_account_id, accounts.id - ORDER BY count(account_id) DESC - OFFSET :offset - LIMIT :limit - SQL - - excluded_account_ids = account.excluded_from_timeline_account_ids + [account.id] - - find_by_sql( - [sql, { account_id: account.id, excluded_account_ids: excluded_account_ids, limit: limit, offset: offset }] - ) - end - def search_for(terms, limit = 10) textsearch, query = generate_query_for_search(terms) diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index ef59f5d15..ee435f956 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -89,10 +89,13 @@ module AccountInteractions .find_or_create_by!(target_account: other_account) rel.update!(show_reblogs: reblogs) + remove_potential_friendship(other_account) + rel end def block!(other_account, uri: nil) + remove_potential_friendship(other_account) block_relationships.create_with(uri: uri) .find_or_create_by!(target_account: other_account) end @@ -100,10 +103,13 @@ module AccountInteractions def mute!(other_account, notifications: nil) notifications = true if notifications.nil? mute = mute_relationships.create_with(hide_notifications: notifications).find_or_create_by!(target_account: other_account) + remove_potential_friendship(other_account) + # When toggling a mute between hiding and allowing notifications, the mute will already exist, so the find_or_create_by! call will return the existing Mute without updating the hide_notifications attribute. Therefore, we check that hide_notifications? is what we want and set it if it isn't. if mute.hide_notifications? != notifications mute.update!(hide_notifications: notifications) end + mute end @@ -194,4 +200,10 @@ module AccountInteractions lists.joins(account: :user) .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago) end + + private + + def remove_potential_friendship(other_account) + PotentialFriendshipTracker.remove(id, other_account.id) + end end diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb index bc2d1547a..6e1ac3ba9 100644 --- a/app/services/favourite_service.rb +++ b/app/services/favourite_service.rb @@ -15,7 +15,10 @@ class FavouriteService < BaseService return favourite unless favourite.nil? favourite = Favourite.create!(account: account, status: status) + create_notification(favourite) + bump_potential_friendship(account, status) + favourite end @@ -33,6 +36,11 @@ class FavouriteService < BaseService end end + def bump_potential_friendship(account, status) + return if account.following?(status.account_id) + PotentialFriendshipTracker.record(account.id, status.account_id, :favourite) + end + def build_json(favourite) Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new( favourite, diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 735985725..bad82051a 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -47,6 +47,8 @@ class PostStatusService < BaseService redis.setex("idempotency:status:#{account.id}:#{options[:idempotency]}", 3_600, status.id) end + bump_potential_friendship(account, status) + status end @@ -79,4 +81,9 @@ class PostStatusService < BaseService def redis Redis.current end + + def bump_potential_friendship(account, status) + return if !status.reply? || account.following?(status.account_id) + PotentialFriendshipTracker.record(account.id, status.in_reply_to_account_id, :reply) + end end diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 3c4e5847f..0ee8bac2f 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -24,6 +24,8 @@ class ReblogService < BaseService ActivityPub::DistributionWorker.perform_async(reblog.id) create_notification(reblog) + bump_potential_friendship(account, reblog) + reblog end @@ -41,6 +43,11 @@ class ReblogService < BaseService end end + def bump_potential_friendship(account, reblog) + return if account.following?(reblog.reblog.account_id) + PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog) + end + def build_json(reblog) Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new( reblog, diff --git a/config/routes.rb b/config/routes.rb index 5fdd3b390..e59325964 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -246,6 +246,7 @@ Rails.application.routes.draw do resources :streaming, only: [:index] resources :custom_emojis, only: [:index] + resources :suggestions, only: [:index] get '/search', to: 'search#index', as: :search diff --git a/spec/controllers/api/v1/suggestions_controller_spec.rb b/spec/controllers/api/v1/suggestions_controller_spec.rb new file mode 100644 index 000000000..17f10b04f --- /dev/null +++ b/spec/controllers/api/v1/suggestions_controller_spec.rb @@ -0,0 +1,35 @@ +require 'rails_helper' + +RSpec.describe Api::V1::SuggestionsController, type: :controller do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let(:bob) { Fabricate(:account) } + let(:jeff) { Fabricate(:account) } + + before do + PotentialFriendshipTracker.record(user.account_id, bob.id, :reblog) + PotentialFriendshipTracker.record(user.account_id, jeff.id, :favourite) + + get :index + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns accounts' do + json = body_as_json + + expect(json.size).to be >= 1 + expect(json.map { |i| i[:id] }).to include *[bob, jeff].map { |i| i.id.to_s } + end + end +end diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index cce659a8a..c50791bcd 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -454,77 +454,6 @@ RSpec.describe Account, type: :model do end end - describe '.triadic_closures' do - let!(:me) { Fabricate(:account) } - let!(:friend) { Fabricate(:account) } - let!(:friends_friend) { Fabricate(:account) } - let!(:both_follow) { Fabricate(:account) } - - before do - me.follow!(friend) - friend.follow!(friends_friend) - - me.follow!(both_follow) - friend.follow!(both_follow) - end - - it 'finds accounts you dont follow which are followed by accounts you do follow' do - expect(described_class.triadic_closures(me)).to eq [friends_friend] - end - - it 'limits by 5 with offset 0 by defualt' do - first_degree = 6.times.map { Fabricate(:account) } - matches = 5.times.map { Fabricate(:account) } - first_degree.each { |account| me.follow!(account) } - matches.each do |match| - first_degree.each { |account| account.follow!(match) } - first_degree.shift - end - - expect(described_class.triadic_closures(me)).to eq matches - end - - it 'accepts arbitrary limits' do - another_friend = Fabricate(:account) - higher_friends_friend = Fabricate(:account) - me.follow!(another_friend) - friend.follow!(higher_friends_friend) - another_friend.follow!(higher_friends_friend) - - expect(described_class.triadic_closures(me, limit: 1)).to eq [higher_friends_friend] - end - - it 'acceps arbitrary offset' do - another_friend = Fabricate(:account) - higher_friends_friend = Fabricate(:account) - me.follow!(another_friend) - friend.follow!(higher_friends_friend) - another_friend.follow!(higher_friends_friend) - - expect(described_class.triadic_closures(me, offset: 1)).to eq [friends_friend] - end - - context 'when you block account' do - before do - me.block!(friends_friend) - end - - it 'rejects blocked accounts' do - expect(described_class.triadic_closures(me)).to be_empty - end - end - - context 'when you mute account' do - before do - me.mute!(friends_friend) - end - - it 'rejects muted accounts' do - expect(described_class.triadic_closures(me)).to be_empty - end - end - end - describe '#statuses_count' do subject { Fabricate(:account) } -- cgit From 4b198b172d0924c2cf9c3f19b3cc6238c140b1f0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 5 Jul 2018 18:43:37 +0200 Subject: Check reblogged status for blocked/muted mentions (#7957) --- app/lib/feed_manager.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index ee9185d34..efe7e2b7b 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -157,7 +157,11 @@ class FeedManager check_for_blocks = status.mentions.pluck(:account_id) check_for_blocks.concat([status.account_id]) - check_for_blocks.concat([status.reblog.account_id]) if status.reblog? + + if status.reblog? + check_for_blocks.concat([status.reblog.account_id]) + check_for_blocks.concat(status.reblog.mentions.pluck(:account_id)) + end return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home) -- cgit From 404c7702ec2a6576ef3b54f6a800624adfda9d53 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 6 Jul 2018 02:15:44 +0200 Subject: In keyword filter, account for reblogs, HTML and whole-words (#7960) * In keyword filter, account for reblogs, HTML and whole-words * Match whole words in JS filter, too * Fix typo --- app/javascript/mastodon/selectors/index.js | 2 +- app/lib/feed_manager.rb | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'app/lib') diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index 56eca1f02..ba9777eba 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -43,7 +43,7 @@ const regexFromFilters = filters => { return null; } - return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).join('|'), 'i'); + return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).map(expr => `\\b${expr}\\b`).join('|'), 'i'); }; export const makeGetStatus = () => { diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index efe7e2b7b..55c72d0ea 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -200,13 +200,14 @@ class FeedManager active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - active_filters.map! { |filter| Regexp.new(Regexp.escape(filter.phrase), true) } + active_filters.map! { |filter| Regexp.new("\\b#{Regexp.escape(filter.phrase)}\\b", true) } return false if active_filters.empty? combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) } + status = status.reblog if status.reblog? - !combined_regex.match(status.text).nil? || + !combined_regex.match(Formatter.instance.plaintext(status)).nil? || (status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?) end -- cgit From 1ca4e51eb38de6de81cedf3ddcdaa626f1d1c569 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 9 Jul 2018 02:22:09 +0200 Subject: Add option to not consider word boundaries when processing keyword filtering (#7975) * Add option to not consider word boundaries when filtering phrases * Add a few tests for keyword/phrase filtering --- app/controllers/api/v1/filters_controller.rb | 2 +- app/javascript/mastodon/selectors/index.js | 5 +++- app/lib/feed_manager.rb | 11 ++++++++- app/models/custom_filter.rb | 1 + app/serializers/rest/filter_serializer.rb | 2 +- app/views/filters/_fields.html.haml | 3 +++ ...180707154237_add_whole_word_to_custom_filter.rb | 17 +++++++++++++ db/schema.rb | 3 ++- spec/lib/feed_manager_spec.rb | 28 +++++++++++++++++----- 9 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 db/migrate/20180707154237_add_whole_word_to_custom_filter.rb (limited to 'app/lib') diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb index a98080d1d..e5ebaff4d 100644 --- a/app/controllers/api/v1/filters_controller.rb +++ b/app/controllers/api/v1/filters_controller.rb @@ -43,6 +43,6 @@ class Api::V1::FiltersController < Api::BaseController end def resource_params - params.permit(:phrase, :expires_in, :irreversible, context: []) + params.permit(:phrase, :expires_in, :irreversible, :whole_word, context: []) end end diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index 7aa7569a0..d0212c379 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -45,7 +45,10 @@ export const regexFromFilters = filters => { return null; } - return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).map(expr => `\\b${expr}\\b`).join('|'), 'i'); + return new RegExp(filters.map(filter => { + let expr = escapeRegExp(filter.get('phrase')); + return filter.get('whole_word') ? `\\b${expr}\\b` : expr; + }).join('|'), 'i'); }; export const makeGetStatus = () => { diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 55c72d0ea..c247ab21d 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -200,7 +200,16 @@ class FeedManager active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - active_filters.map! { |filter| Regexp.new("\\b#{Regexp.escape(filter.phrase)}\\b", true) } + active_filters.map! do |filter| + if filter.whole_word + sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : '' + eb = filter.phrase =~ /[[:word:]]\Z/ ? '\b' : '' + + /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ + else + /#{Regexp.escape(filter.phrase)}/i + end + end return false if active_filters.empty? diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 2c1a54375..342207a55 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -8,6 +8,7 @@ # expires_at :datetime # phrase :text default(""), not null # context :string default([]), not null, is an Array +# whole_word :boolean default(TRUE), not null # irreversible :boolean default(FALSE), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/serializers/rest/filter_serializer.rb b/app/serializers/rest/filter_serializer.rb index 51340aa79..3134be371 100644 --- a/app/serializers/rest/filter_serializer.rb +++ b/app/serializers/rest/filter_serializer.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class REST::FilterSerializer < ActiveModel::Serializer - attributes :id, :phrase, :context, :expires_at, + attributes :id, :phrase, :context, :whole_word, :expires_at, :irreversible end diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml index af5d648b8..a5a3f0337 100644 --- a/app/views/filters/_fields.html.haml +++ b/app/views/filters/_fields.html.haml @@ -7,5 +7,8 @@ .fields-group = f.input :irreversible, wrapper: :with_label +.fields-group + = f.input :whole_word, wrapper: :with_label + .fields-group = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, prompt: I18n.t('invites.expires_in_prompt') diff --git a/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb b/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb new file mode 100644 index 000000000..63ecb8741 --- /dev/null +++ b/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb @@ -0,0 +1,17 @@ +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class AddWholeWordToCustomFilter < ActiveRecord::Migration[5.2] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def change + safety_assured do + add_column_with_default :custom_filters, :whole_word, :boolean, default: true, allow_null: false + end + end + + def down + remove_column :custom_filters, :whole_word + end +end diff --git a/db/schema.rb b/db/schema.rb index 661fc8179..02032c548 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_06_28_181026) do +ActiveRecord::Schema.define(version: 2018_07_07_154237) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -149,6 +149,7 @@ ActiveRecord::Schema.define(version: 2018_06_28_181026) do t.text "phrase", default: "", null: false t.string "context", default: [], null: false, array: true t.boolean "irreversible", default: false, null: false + t.boolean "whole_word", default: true, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_custom_filters_on_account_id" diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index d1b847675..7535e144d 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -127,12 +127,28 @@ RSpec.describe FeedManager do expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true end - it 'returns true if status contains irreversibly muted phrase' do - alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) - alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + context 'for irreversibly muted phrases' do + it 'considers word boundaries when matching' do + alice.custom_filters.create!(phrase: 'bob', context: %w(home), irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'bobcats', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be_falsy + end + + it 'returns true if phrase is contained' do + alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) + alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + end + + it 'matches substrings if whole_word is false' do + alice.custom_filters.create!(phrase: 'take', context: %w(home), whole_word: false, irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'shiitake', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + end end end -- cgit