From cdb101340a20183a82889f811d9311c370c855e5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Jun 2018 15:34:36 +0200 Subject: Keyword/phrase filtering (#7905) * Add keyword filtering GET|POST /api/v1/filters GET|PUT|DELETE /api/v1/filters/:id - Irreversible filters can drop toots from home or notifications - Other filters can hide toots through the client app - Filters use a phrase valid in particular contexts, expiration * Make sure expired filters don't get applied client-side * Add missing API methods * Remove "regex filter" from column settings * Add tests * Add test for FeedManager * Add CustomFilter test * Add UI for managing filters * Add streaming API event to allow syncing filters * Fix tests --- config/routes.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'config/routes.rb') diff --git a/config/routes.rb b/config/routes.rb index a3cba24fc..5fdd3b390 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -114,6 +114,7 @@ Rails.application.routes.draw do resources :tags, only: [:show] resources :emojis, only: [:show] resources :invites, only: [:index, :create, :destroy] + resources :filters, except: [:show] get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy @@ -254,6 +255,7 @@ Rails.application.routes.draw do resources :mutes, only: [:index] resources :favourites, only: [:index] resources :reports, only: [:index, :create] + resources :filters, only: [:index, :create, :show, :update, :destroy] namespace :apps do get :verify_credentials, to: 'credentials#show' -- cgit From da8fe8079e13758f45e5ba77cb8023c554ae193c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 3 Jul 2018 01:47:56 +0200 Subject: Re-add follow recommendations API (#7918) * Re-add follow recommendations API GET /api/v1/suggestions Removed in 8efa081f210d72ed450c39ac4cde0fd84fb3d3fb due to Neo4J dependency. The algorithm uses triadic closures, takes into account suspensions, blocks, mutes, domain blocks, excludes locked and moved accounts, and prefers more recently updated accounts. * Track interactions with people you don't follow Replying to, favouriting and reblogging someone you're not following will make them show up in follow recommendations. The interactions have different weights: - Replying is 1 - Favouriting is 10 (decidedly positive interaction, but private) - Reblogging is 20 Following them, muting or blocking will remove them from the list, obviously. * Remove triadic closures, ensure potential friendships are trimmed --- app/controllers/api/v1/suggestions_controller.rb | 21 +++++++ app/lib/potential_friendship_tracker.rb | 39 ++++++++++++ app/models/account.rb | 29 +-------- app/models/concerns/account_interactions.rb | 12 ++++ app/services/favourite_service.rb | 8 +++ app/services/post_status_service.rb | 7 +++ app/services/reblog_service.rb | 7 +++ config/routes.rb | 1 + .../api/v1/suggestions_controller_spec.rb | 35 +++++++++++ spec/models/account_spec.rb | 71 ---------------------- 10 files changed, 131 insertions(+), 99 deletions(-) create mode 100644 app/controllers/api/v1/suggestions_controller.rb create mode 100644 app/lib/potential_friendship_tracker.rb create mode 100644 spec/controllers/api/v1/suggestions_controller_spec.rb (limited to 'config/routes.rb') diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb new file mode 100644 index 000000000..3abccedd5 --- /dev/null +++ b/app/controllers/api/v1/suggestions_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::SuggestionsController < Api::BaseController + include Authorization + + before_action -> { doorkeeper_authorize! :read } + before_action :require_user! + before_action :set_accounts + + respond_to :json + + def index + render json: @accounts, each_serializer: REST::AccountSerializer + end + + private + + def set_accounts + @accounts = PotentialFriendshipTracker.get(current_account.id, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT)) + end +end diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb new file mode 100644 index 000000000..362482669 --- /dev/null +++ b/app/lib/potential_friendship_tracker.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class PotentialFriendshipTracker + EXPIRE_AFTER = 90.days.seconds + MAX_ITEMS = 80 + + WEIGHTS = { + reply: 1, + favourite: 10, + reblog: 20, + }.freeze + + class << self + def record(account_id, target_account_id, action) + key = "interactions:#{account_id}" + weight = WEIGHTS[action] + + redis.zincrby(key, weight, target_account_id) + redis.zremrangebyrank(key, 0, -MAX_ITEMS) + redis.expire(key, EXPIRE_AFTER) + end + + def remove(account_id, target_account_id) + redis.zrem("interactions:#{account_id}", target_account_id) + end + + def get(account_id, limit: 20, offset: 0) + account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit) + return [] if account_ids.empty? + Account.searchable.where(id: account_ids) + end + + private + + def redis + Redis.current + end + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 40a45b1f8..1f720bf88 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -127,6 +127,7 @@ class Account < ApplicationRecord scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) } scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) } scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } + scope :searchable, -> { where(suspended: false).where(moved_to_account_id: nil) } delegate :email, :unconfirmed_email, @@ -309,34 +310,6 @@ class Account < ApplicationRecord DeliveryFailureTracker.filter(urls) end - def triadic_closures(account, limit: 5, offset: 0) - sql = <<-SQL.squish - WITH first_degree AS ( - SELECT target_account_id - FROM follows - WHERE account_id = :account_id - ) - SELECT accounts.* - FROM follows - INNER JOIN accounts ON follows.target_account_id = accounts.id - WHERE - account_id IN (SELECT * FROM first_degree) - AND target_account_id NOT IN (SELECT * FROM first_degree) - AND target_account_id NOT IN (:excluded_account_ids) - AND accounts.suspended = false - GROUP BY target_account_id, accounts.id - ORDER BY count(account_id) DESC - OFFSET :offset - LIMIT :limit - SQL - - excluded_account_ids = account.excluded_from_timeline_account_ids + [account.id] - - find_by_sql( - [sql, { account_id: account.id, excluded_account_ids: excluded_account_ids, limit: limit, offset: offset }] - ) - end - def search_for(terms, limit = 10) textsearch, query = generate_query_for_search(terms) diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index ef59f5d15..ee435f956 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -89,10 +89,13 @@ module AccountInteractions .find_or_create_by!(target_account: other_account) rel.update!(show_reblogs: reblogs) + remove_potential_friendship(other_account) + rel end def block!(other_account, uri: nil) + remove_potential_friendship(other_account) block_relationships.create_with(uri: uri) .find_or_create_by!(target_account: other_account) end @@ -100,10 +103,13 @@ module AccountInteractions def mute!(other_account, notifications: nil) notifications = true if notifications.nil? mute = mute_relationships.create_with(hide_notifications: notifications).find_or_create_by!(target_account: other_account) + remove_potential_friendship(other_account) + # When toggling a mute between hiding and allowing notifications, the mute will already exist, so the find_or_create_by! call will return the existing Mute without updating the hide_notifications attribute. Therefore, we check that hide_notifications? is what we want and set it if it isn't. if mute.hide_notifications? != notifications mute.update!(hide_notifications: notifications) end + mute end @@ -194,4 +200,10 @@ module AccountInteractions lists.joins(account: :user) .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago) end + + private + + def remove_potential_friendship(other_account) + PotentialFriendshipTracker.remove(id, other_account.id) + end end diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb index bc2d1547a..6e1ac3ba9 100644 --- a/app/services/favourite_service.rb +++ b/app/services/favourite_service.rb @@ -15,7 +15,10 @@ class FavouriteService < BaseService return favourite unless favourite.nil? favourite = Favourite.create!(account: account, status: status) + create_notification(favourite) + bump_potential_friendship(account, status) + favourite end @@ -33,6 +36,11 @@ class FavouriteService < BaseService end end + def bump_potential_friendship(account, status) + return if account.following?(status.account_id) + PotentialFriendshipTracker.record(account.id, status.account_id, :favourite) + end + def build_json(favourite) Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new( favourite, diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 735985725..bad82051a 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -47,6 +47,8 @@ class PostStatusService < BaseService redis.setex("idempotency:status:#{account.id}:#{options[:idempotency]}", 3_600, status.id) end + bump_potential_friendship(account, status) + status end @@ -79,4 +81,9 @@ class PostStatusService < BaseService def redis Redis.current end + + def bump_potential_friendship(account, status) + return if !status.reply? || account.following?(status.account_id) + PotentialFriendshipTracker.record(account.id, status.in_reply_to_account_id, :reply) + end end diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 3c4e5847f..0ee8bac2f 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -24,6 +24,8 @@ class ReblogService < BaseService ActivityPub::DistributionWorker.perform_async(reblog.id) create_notification(reblog) + bump_potential_friendship(account, reblog) + reblog end @@ -41,6 +43,11 @@ class ReblogService < BaseService end end + def bump_potential_friendship(account, reblog) + return if account.following?(reblog.reblog.account_id) + PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog) + end + def build_json(reblog) Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new( reblog, diff --git a/config/routes.rb b/config/routes.rb index 5fdd3b390..e59325964 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -246,6 +246,7 @@ Rails.application.routes.draw do resources :streaming, only: [:index] resources :custom_emojis, only: [:index] + resources :suggestions, only: [:index] get '/search', to: 'search#index', as: :search diff --git a/spec/controllers/api/v1/suggestions_controller_spec.rb b/spec/controllers/api/v1/suggestions_controller_spec.rb new file mode 100644 index 000000000..17f10b04f --- /dev/null +++ b/spec/controllers/api/v1/suggestions_controller_spec.rb @@ -0,0 +1,35 @@ +require 'rails_helper' + +RSpec.describe Api::V1::SuggestionsController, type: :controller do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let(:bob) { Fabricate(:account) } + let(:jeff) { Fabricate(:account) } + + before do + PotentialFriendshipTracker.record(user.account_id, bob.id, :reblog) + PotentialFriendshipTracker.record(user.account_id, jeff.id, :favourite) + + get :index + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns accounts' do + json = body_as_json + + expect(json.size).to be >= 1 + expect(json.map { |i| i[:id] }).to include *[bob, jeff].map { |i| i.id.to_s } + end + end +end diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index cce659a8a..c50791bcd 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -454,77 +454,6 @@ RSpec.describe Account, type: :model do end end - describe '.triadic_closures' do - let!(:me) { Fabricate(:account) } - let!(:friend) { Fabricate(:account) } - let!(:friends_friend) { Fabricate(:account) } - let!(:both_follow) { Fabricate(:account) } - - before do - me.follow!(friend) - friend.follow!(friends_friend) - - me.follow!(both_follow) - friend.follow!(both_follow) - end - - it 'finds accounts you dont follow which are followed by accounts you do follow' do - expect(described_class.triadic_closures(me)).to eq [friends_friend] - end - - it 'limits by 5 with offset 0 by defualt' do - first_degree = 6.times.map { Fabricate(:account) } - matches = 5.times.map { Fabricate(:account) } - first_degree.each { |account| me.follow!(account) } - matches.each do |match| - first_degree.each { |account| account.follow!(match) } - first_degree.shift - end - - expect(described_class.triadic_closures(me)).to eq matches - end - - it 'accepts arbitrary limits' do - another_friend = Fabricate(:account) - higher_friends_friend = Fabricate(:account) - me.follow!(another_friend) - friend.follow!(higher_friends_friend) - another_friend.follow!(higher_friends_friend) - - expect(described_class.triadic_closures(me, limit: 1)).to eq [higher_friends_friend] - end - - it 'acceps arbitrary offset' do - another_friend = Fabricate(:account) - higher_friends_friend = Fabricate(:account) - me.follow!(another_friend) - friend.follow!(higher_friends_friend) - another_friend.follow!(higher_friends_friend) - - expect(described_class.triadic_closures(me, offset: 1)).to eq [friends_friend] - end - - context 'when you block account' do - before do - me.block!(friends_friend) - end - - it 'rejects blocked accounts' do - expect(described_class.triadic_closures(me)).to be_empty - end - end - - context 'when you mute account' do - before do - me.mute!(friends_friend) - end - - it 'rejects muted accounts' do - expect(described_class.triadic_closures(me)).to be_empty - end - end - end - describe '#statuses_count' do subject { Fabricate(:account) } -- cgit From 6b9e03e002ed349fd8b7e2879d599bb75a698eb2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 7 Jul 2018 21:09:54 +0200 Subject: Add API method to remove a suggestion (#7978) DELETE /api/v1/suggestions/:account_id When blocking, remove suggestion from both sides. Muting not affected, since muting is supposed to be invisible to the target. --- app/controllers/api/v1/suggestions_controller.rb | 5 +++++ app/models/concerns/account_interactions.rb | 3 ++- config/routes.rb | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) (limited to 'config/routes.rb') diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb index 3abccedd5..9da2b60ae 100644 --- a/app/controllers/api/v1/suggestions_controller.rb +++ b/app/controllers/api/v1/suggestions_controller.rb @@ -13,6 +13,11 @@ class Api::V1::SuggestionsController < Api::BaseController render json: @accounts, each_serializer: REST::AccountSerializer end + def destroy + PotentialFriendshipTracker.remove(current_account.id, params[:id]) + render_empty + end + private def set_accounts diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index ee435f956..e14e041f6 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -203,7 +203,8 @@ module AccountInteractions private - def remove_potential_friendship(other_account) + def remove_potential_friendship(other_account, mutual = false) PotentialFriendshipTracker.remove(id, other_account.id) + PotentialFriendshipTracker.remove(other_account.id, id) if mutual end end diff --git a/config/routes.rb b/config/routes.rb index e59325964..fd26b4aa7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -246,7 +246,7 @@ Rails.application.routes.draw do resources :streaming, only: [:index] resources :custom_emojis, only: [:index] - resources :suggestions, only: [:index] + resources :suggestions, only: [:index, :destroy] get '/search', to: 'search#index', as: :search -- cgit