diff options
Diffstat (limited to 'app/lib/feed_manager.rb')
| -rw-r--r-- | app/lib/feed_manager.rb | 240 |
1 files changed, 184 insertions, 56 deletions
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 3c1f8d6e2..665869b26 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -1,18 +1,17 @@ # frozen_string_literal: true require 'singleton' - class FeedManager include Singleton include Redisable # Maximum number of items stored in a single feed - MAX_ITEMS = 400 + MAX_ITEMS = 1000 # Number of items in the feed since last reblog of status # before the new reblog will be inserted. Must be <= MAX_ITEMS # or the tracking sets will grow forever - REBLOG_FALLOFF = 40 + REBLOG_FALLOFF = 50 # Execute block for every active account # @yield [Account] @@ -40,9 +39,9 @@ class FeedManager def filter?(timeline_type, status, receiver) case timeline_type when :home - filter_from_home?(status, receiver.id, build_crutches(receiver.id, [status])) + filter_from_home?(status, receiver.id, build_crutches(receiver.id, [status]), receiver.user&.filters_unknown?) when :list - filter_from_list?(status, receiver) || filter_from_home?(status, receiver.account_id, build_crutches(receiver.account_id, [status])) + filter_from_list?(status, receiver) || filter_from_home?(status, receiver.account_id, build_crutches(receiver.account_id, [status]), receiver.account.user&.filters_unknown?) when :mentions filter_from_mentions?(status, receiver.id) when :direct @@ -57,7 +56,7 @@ class FeedManager # @param [Status] status # @return [Boolean] def push_to_home(account, status) - return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?) + return false unless add_to_feed(:home, account.id, status, account.user&.home_reblogs?) trim(:home, account.id) PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") @@ -68,8 +67,8 @@ class FeedManager # @param [Account] account # @param [Status] status # @return [Boolean] - def unpush_from_home(account, status) - return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?) + def unpush_from_home(account, status, include_reblogs_list = true) + return false unless remove_from_feed(:home, account.id, status, include_reblogs_list) redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true @@ -80,7 +79,8 @@ class FeedManager # @param [Status] status # @return [Boolean] def push_to_list(list, status) - return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + return false if filter_from_list?(status, list) + return false unless add_to_feed(:list, list.id, status, list.reblogs?) trim(:list, list.id) PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") @@ -92,7 +92,7 @@ class FeedManager # @param [Status] status # @return [Boolean] def unpush_from_list(list, status) - return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + return false unless remove_from_feed(:list, list.id, status) redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true @@ -121,13 +121,33 @@ class FeedManager true end + def unpush_status(account, status) + return if account.blank? || status.blank? + + unpush_from_home(account, status) + unpush_from_direct(account, status) if status.direct_visibility? + + account.lists_for_local_distribution.select(:id, :account_id).each do |list| + unpush_from_list(list, status) + end + end + + def unpush_conversation(account, conversation) + return if account.blank? || conversation.blank? + + conversation.statuses.reorder(nil).find_each do |status| + unpush_status(account, status) + end + end + # Fill a home feed with an account's statuses # @param [Account] from_account # @param [Account] into_account # @return [void] def merge_into_home(from_account, into_account) timeline_key = key(:home, into_account.id) - aggregate = into_account.user&.aggregates_reblogs? + reblogs = into_account.user&.home_reblogs? + no_unknown = into_account.user&.filters_unknown? query = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4) if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4 @@ -139,9 +159,9 @@ class FeedManager crutches = build_crutches(into_account.id, statuses) statuses.each do |status| - next if filter_from_home?(status, into_account.id, crutches) + next if filter_from_home?(status, into_account.id, crutches, no_unknown) - add_to_feed(:home, into_account.id, status, aggregate) + add_to_feed(:home, into_account.id, status, reblogs) end trim(:home, into_account.id) @@ -153,7 +173,8 @@ class FeedManager # @return [void] def merge_into_list(from_account, list) timeline_key = key(:list, list.id) - aggregate = list.account.user&.aggregates_reblogs? + reblogs = list.account.user&.home_reblogs? + no_unknown = list.account.user&.filters_unknown? query = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4) if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4 @@ -165,9 +186,9 @@ class FeedManager crutches = build_crutches(list.account_id, statuses) statuses.each do |status| - next if filter_from_home?(status, list.account_id, crutches) || filter_from_list?(status, list) + next if filter_from_home?(status, list.account_id, crutches, no_unknown) || filter_from_list?(status, list) - add_to_feed(:list, list.id, status, aggregate) + add_to_feed(:list, list.id, status, reblogs) end trim(:list, list.id) @@ -182,7 +203,7 @@ class FeedManager oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| - remove_from_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?) + remove_from_feed(:home, into_account.id, status) end end @@ -195,7 +216,7 @@ class FeedManager oldest_list_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_list_score).reorder(nil).find_each do |status| - remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + remove_from_feed(:list, list.id, status, !list.reblogs?) end end @@ -219,16 +240,63 @@ class FeedManager end end + # Clear all reblogs from a home feed + # @param [Account] account + # @return [void] + def clear_reblogs_from_home(account) + timeline_key = key(:home, account.id) + timeline_status_ids = redis.zrange(timeline_key, 0, -1) + + Status.reblogs.joins(:reblog).where(reblogs_statuses: { local: false }).where(id: timeline_status_ids).find_each do |status| + unpush_from_home(account, status, false) + end + end + + # Populate list feeds of account from scratch + # @param [Account] account + # @return [void] + def populate_lists(account) + limit = FeedManager::MAX_ITEMS / 2 + + account.owned_lists.includes(:accounts) do |list| + timeline_key = key(:list, list.id) + + list.accounts.includes(:account_stat).find_each do |target_account| + if redis.zcard(timeline_key) >= limit + oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i + last_status_score = Mastodon::Snowflake.id_at(account.last_status_at) + + # If the feed is full and this account has not posted more recently + # than the last item on the feed, then we can skip the whole account + # because none of its statuses would stay on the feed anyway + next if last_status_score < oldest_home_score + end + + statuses = target_account.statuses.published.without_reblogs.where(visibility: [:public, :unlisted, :private]).includes(:mentions, :preloadable_poll).limit(limit) + crutches = build_crutches(account.id, statuses) + + statuses.each do |status| + next if filter_from_list?(status, account.id) || filter_from_home?(status, account.id, crutches, account.user&.filters_unknown?) + + add_to_feed(:list, list.id, status, list.reblogs?) + end + + trim(:list, list.id) + end + end + end + # Populate home feed of account from scratch # @param [Account] account # @return [void] def populate_home(account) limit = FeedManager::MAX_ITEMS / 2 - aggregate = account.user&.aggregates_reblogs? + reblogs = account.user&.home_reblogs? + no_unknown = account.user&.filters_unknown? timeline_key = key(:home, account.id) account.statuses.limit(limit).each do |status| - add_to_feed(:home, account.id, status, aggregate) + add_to_feed(:home, account.id, status, reblogs) end account.following.includes(:account_stat).find_each do |target_account| @@ -242,13 +310,13 @@ class FeedManager next if last_status_score < oldest_home_score end - statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(limit) + statuses = target_account.statuses.published.where(visibility: [:public, :unlisted, :private]).includes(:mentions, :preloadable_poll, reblog: [:account, :mentions]).limit(limit) crutches = build_crutches(account.id, statuses) statuses.each do |status| - next if filter_from_home?(status, account.id, crutches) + next if filter_from_home?(status, account.id, crutches, no_unknown) - add_to_feed(:home, account.id, status, aggregate) + add_to_feed(:home, account.id, status, reblogs, false) end trim(:home, account.id) @@ -270,6 +338,7 @@ class FeedManager statuses.each do |status| next if filter_from_direct?(status, account) + added += 1 if add_to_feed(:direct, account.id, status) end @@ -334,36 +403,55 @@ class FeedManager # @param [Integer] receiver_id # @param [Hash] crutches # @return [Boolean] - def filter_from_home?(status, receiver_id, crutches) + def filter_from_home?(status, receiver_id, crutches, followed_only = false) return false if receiver_id == status.account_id + return true if !status.published? || crutches[:hiding_thread][status.conversation_id] return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) return true if phrase_filtered?(status, receiver_id, :home) check_for_blocks = crutches[:active_mentions][status.id] || [] check_for_blocks.concat([status.account_id]) + check_for_blocks.concat([status.in_reply_to_account_id]) if status.reply? if status.reblog? check_for_blocks.concat([status.reblog.account_id]) check_for_blocks.concat(crutches[:active_mentions][status.reblog_of_id] || []) + check_for_blocks.concat([status.reblog.in_reply_to_account_id]) if status.reblog.reply? end return true if check_for_blocks.any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } - if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply - should_filter = !crutches[:following][status.in_reply_to_account_id] # and I'm not following the person it's a reply to - should_filter &&= receiver_id != status.in_reply_to_account_id # and it's not a reply to me - should_filter &&= status.account_id != status.in_reply_to_account_id # and it's not a self-reply + # Filter if... + if status.reply? # ...it's a reply and... + # ...you're not following the participants... + should_filter = (status.mentions.pluck(:account_id) - crutches[:following].keys).present? + # ...and the author isn't replying to you... + should_filter &&= receiver_id != status.in_reply_to_account_id return !!should_filter - elsif status.reblog? # Filter out a reblog - should_filter = crutches[:hiding_reblogs][status.account_id] # if the reblogger's reblogs are suppressed - should_filter ||= crutches[:blocked_by][status.reblog.account_id] # or if the author of the reblogged status is blocking me - should_filter ||= crutches[:domain_blocking][status.reblog.account.domain] # or the author's domain is blocked + elsif status.reblog? # ...it's a boost and... + # ...you don't follow the OP and they're non-local or they're silenced... + should_filter = (followed_only || status.reblog.account.silenced?) && !crutches[:following][status.reblog.account_id] + + # ..or you're hiding boosts from them... + should_filter ||= crutches[:hiding_reblogs][status.account_id] + # ...or they're blocking you... + should_filter ||= crutches[:blocked_by][status.reblog.account_id] + # ...or you're blocking their domain... + should_filter ||= crutches[:domain_blocking][status.reblog.account.domain] + + # ...or it's a reply... + if !(should_filter || status.reblog.in_reply_to_account_id.nil?) && status.reblog.reply? + # ...and you don't follow the participants... + should_filter ||= (status.reblog.mentions.pluck(:account_id) - crutches[:following].keys).present? + # ...and the author isn't replying to you... + should_filter &&= receiver_id != status.in_reply_to_account_id + end return !!should_filter end - false + !crutches[:following][status.account_id] end # Check if status should not be added to the mentions feed @@ -381,18 +469,29 @@ class FeedManager check_for_blocks = status.active_mentions.pluck(:account_id) check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil? - should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted) - should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them + should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) + should_filter ||= (status.account.silenced? && !relationship_exists?(receiver_id, status.account_id)) should_filter end + def following?(account_id, target_account_id) + Follow.where(account_id: account_id, target_account_id: target_account_id).exists? + end + + def relationship_exists?(account_id, target_account_id) + Follow.where(account_id: account_id, target_account_id: target_account_id) + .or(Follow.where(account_id: target_account_id, target_account_id: account_id)) + .exists? + end + # Check if status should not be added to the linear direct message feed # @param [Status] status # @param [Integer] receiver_id # @return [Boolean] def filter_from_direct?(status, receiver_id) return false if receiver_id == status.account_id + filter_from_mentions?(status, receiver_id) end @@ -401,6 +500,9 @@ class FeedManager # @param [List] list # @return [Boolean] def filter_from_list?(status, list) + return true if (list.reblogs? && !status.reblog?) || (!list.reblogs? && status.reblog?) + return true if status.reblog? ? status.reblog.account_id == list.account_id : status.account_id == list.account_id + if status.reply? && status.in_reply_to_account_id != status.account_id should_filter = status.in_reply_to_account_id != list.account_id should_filter &&= !list.show_all_replies? @@ -455,13 +557,19 @@ class FeedManager # @param [Symbol] timeline_type # @param [Integer] account_id # @param [Status] status - # @param [Boolean] aggregate_reblogs + # @param [Boolean] home_reblogs + # @param [Boolean] stream # @return [Boolean] - def add_to_feed(timeline_type, account_id, status, aggregate_reblogs = true) + def add_to_feed(timeline_type, account_id, status, home_reblogs = true, stream = true) timeline_key = key(timeline_type, account_id) reblog_key = key(timeline_type, account_id, 'reblogs') - if status.reblog? && (aggregate_reblogs.nil? || aggregate_reblogs) + if status.reblog? + add_to_reblogs(account_id, status, stream) if timeline_type == :home + return false unless home_reblogs || (timeline_type == :home && (status.reblog.local? || following?(account_id, status.reblog.account_id))) + end + + if status.reblog? # If the original status or a reblog of it is within # REBLOG_FALLOFF statuses from the top, do not re-insert it into # the feed @@ -493,6 +601,8 @@ class FeedManager redis.zadd(timeline_key, status.id, status.id) end + add_to_reblogs(account_id, status, stream) if timeline_type == :home && status.reblog? + true end @@ -503,13 +613,15 @@ class FeedManager # @param [Symbol] timeline_type # @param [Integer] account_id # @param [Status] status - # @param [Boolean] aggregate_reblogs + # @param [Boolean] include_reblogs_list # @return [Boolean] - def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs = true) + def remove_from_feed(timeline_type, account_id, status, include_reblogs_list = true) timeline_key = key(timeline_type, account_id) reblog_key = key(timeline_type, account_id, 'reblogs') - if status.reblog? && (aggregate_reblogs.nil? || aggregate_reblogs) + remove_from_reblogs(account_id, status) if include_reblogs_list && timeline_type == :home && status.reblog? + + if status.reblog? # 1. If the reblogging status is not in the feed, stop. status_rank = redis.zrevrank(timeline_key, status.id) return false if status_rank.nil? @@ -547,27 +659,43 @@ class FeedManager def build_crutches(receiver_id, statuses) crutches = {} - crutches[:active_mentions] = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id).each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) } - - check_for_blocks = statuses.flat_map do |s| - arr = crutches[:active_mentions][s.id] || [] - arr.concat([s.account_id]) - - if s.reblog? - arr.concat([s.reblog.account_id]) - arr.concat(crutches[:active_mentions][s.reblog_of_id] || []) - end + mentions = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id) + participants = statuses.flat_map { |s| [s.account_id, s.in_reply_to_account_id, s.reblog&.account_id, s.reblog&.in_reply_to_account_id].compact } | mentions.map { |m| m[1] } - arr - end + crutches[:active_mentions] = mentions.each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) } - crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: participants).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: participants).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: participants).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true } crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:hiding_thread] = ConversationMute.where(account_id: receiver_id, conversation_id: statuses.map(&:conversation_id).compact).pluck(:conversation_id).each_with_object({}) { |id, mapping| mapping[id] = true } crutches end + + def find_or_create_reblogs_list(account_id) + List.find_or_create_by!(account_id: account_id, reblogs: true) do |list| + list.title = I18n.t('accounts.reblogs') + list.replies_policy = :no_replies + end + end + + def add_to_reblogs(account_id, status, stream = true) + reblogs_list_id = find_or_create_reblogs_list(account_id).id + return unless add_to_feed(:list, reblogs_list_id, status) + + trim(:list, reblogs_list_id) + return unless stream && push_update_required?("timeline:list:#{reblogs_list_id}") + + PushUpdateWorker.perform_async(account_id, status.id, "timeline:list:#{reblogs_list_id}") + end + + def remove_from_reblogs(account_id, status) + reblogs_list_id = find_or_create_reblogs_list(account_id).id + return unless remove_from_feed(:list, reblogs_list_id, status) + + redis.publish("timeline:list:#{reblogs_list_id}", Oj.dump(event: :delete, payload: status.id.to_s)) + end end |