From 554c2fd8af79c173e81d7193ea649fa848076123 Mon Sep 17 00:00:00 2001 From: aschmitz Date: Tue, 17 Oct 2017 04:45:06 -0500 Subject: Clean up reblog tracking keys, related improvements (#5428) * Clean up reblog-tracking sets from FeedManager Builds on #5419, with a few minor optimizations and cleanup of sets after they are no longer needed. * Update tests, fix multiply-reblogged case Previously, we would have lost the fact that a given status was reblogged if the displayed reblog of it was removed, now we don't. Also added tests to make sure FeedManager#trim cleans up our reblog tracking keys, fixed up FeedCleanupScheduler to use the right loop, and fixed the test for it. --- app/lib/feed_manager.rb | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) (limited to 'app/lib') diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 7f95d7784..58650efb6 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -56,7 +56,17 @@ class FeedManager falloff_rank = FeedManager::REBLOG_FALLOFF - 1 falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true) falloff_score = falloff_range&.first&.last&.to_i || 0 - redis.zremrangebyscore(reblog_key, 0, falloff_score) + + # Get any reblogs we might have to clean up after. + redis.zrangebyscore(reblog_key, 0, falloff_score).each do |reblogged_id| + # Remove it from the set of reblogs we're tracking *first* to avoid races. + redis.zrem(reblog_key, reblogged_id) + # Just drop any set we might have created to track additional reblogs. + # This means that if this reblog is deleted, we won't automatically insert + # another reblog, but also that any new reblog can be inserted into the + # feed. + redis.del(key(type, account_id, "reblogs:#{reblogged_id}")) + end end def push_update_required?(timeline_type, account_id) @@ -177,23 +187,28 @@ class FeedManager reblog_key = key(timeline_type, account.id, 'reblogs') if status.reblog? - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") - # If the original status or a reblog of it is within # REBLOG_FALLOFF statuses from the top, do not re-insert it into # the feed rank = redis.zrevrank(timeline_key, status.reblog_of_id) - redis.sadd(reblog_set_key, status.reblog_of_id) unless rank.nil? - redis.sadd(reblog_set_key, status.id) - return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id) - return false unless reblog_rank.nil? - - redis.zadd(timeline_key, status.id, status.id) - redis.zadd(reblog_key, status.id, status.reblog_of_id) + if reblog_rank.nil? + # This is not something we've already seen reblogged, so we + # can just add it to the feed (and note that we're + # reblogging it). + redis.zadd(timeline_key, status.id, status.id) + redis.zadd(reblog_key, status.id, status.reblog_of_id) + else + # Another reblog of the same status was already in the + # REBLOG_FALLOFF most recent statuses, so we note that this + # is an "extra" reblog, by storing it in reblog_set_key. + reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + redis.sadd(reblog_set_key, status.id) + return false + end else redis.zadd(timeline_key, status.id, status.id) end @@ -207,23 +222,21 @@ class FeedManager # do so if appropriate. def remove_from_feed(timeline_type, account, status) timeline_key = key(timeline_type, account.id) - reblog_key = key(timeline_type, account.id, 'reblogs') if status.reblog? # 1. If the reblogging status is not in the feed, stop. status_rank = redis.zrevrank(timeline_key, status.id) return false if status_rank.nil? - # 2. Remove the reblogged status from the `:reblogs` zset. - redis.zrem(reblog_key, status.reblog_of_id) - - # 3. Remove reblog from set of this status's reblogs, and - # re-insert another reblog or original into the feed if - # one remains in the set + # 2. Remove reblog from set of this status's reblogs. reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") redis.srem(reblog_set_key, status.id) - other_reblog = redis.srandmember(reblog_set_key) + # 3. Re-insert another reblog or original into the feed if one + # remains in the set. We could pick a random element, but this + # set should generally be small, and it seems ideal to show the + # oldest potential such reblog. + other_reblog = redis.smembers(reblog_set_key).map(&:to_i).sort.first redis.zadd(timeline_key, other_reblog, other_reblog) if other_reblog -- cgit From 484208ce12402d8b82070038d9be8441b82f86e7 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 17 Oct 2017 20:05:21 +0200 Subject: When status is fetched instead of delivered, do not stream it (#5437) --- app/lib/activitypub/activity.rb | 7 ++++++- app/lib/ostatus/activity/creation.rb | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 9688f57a6..01144f595 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -67,9 +67,14 @@ class ActivityPub::Activity end def distribute(status) + crawl_links(status) + + # Only continue if the status is supposed to have + # arrived in real-time + return unless @options[:override_timestamps] + notify_about_reblog(status) if reblog_of_local_account?(status) notify_about_mentions(status) - crawl_links(status) distribute_to_followers(status) end diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb index a1ab522e2..3418e2420 100644 --- a/app/lib/ostatus/activity/creation.rb +++ b/app/lib/ostatus/activity/creation.rb @@ -56,7 +56,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? - DistributionWorker.perform_async(status.id) + DistributionWorker.perform_async(status.id) if @options[:override_timestamps] status end -- cgit