change sidekiq queueing to bulk push (#3536)

6 files changed, 23 insertions, 9 deletions

diff --git a/Gemfile b/Gemfile
index 835a6996a..d5fbab748 100644
--- a/Gemfile
+++ b/Gemfile
@@ -52,6 +52,7 @@ gem 'sanitize', '~> 4.4'
 gem 'sidekiq', '~> 5.0'
 gem 'sidekiq-scheduler', '~> 2.1'
 gem 'sidekiq-unique-jobs', '~> 5.0'
+gem 'sidekiq-bulk', '~>0.1.1'
 gem 'simple-navigation', '~> 4.0'
 gem 'simple_form', '~> 3.4'
 gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'
diff --git a/Gemfile.lock b/Gemfile.lock
index c7ea6d79e..ee86bfafd 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -402,6 +402,9 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.3, >= 3.3.3)
+    sidekiq-bulk (0.1.1)
+      activesupport
+      sidekiq
     sidekiq-scheduler (2.1.4)
       redis (~> 3)
       rufus-scheduler (~> 3.2)
@@ -543,6 +546,7 @@ DEPENDENCIES
   sanitize (~> 4.4)
   scss_lint (~> 0.53)
   sidekiq (~> 5.0)
+  sidekiq-bulk (~> 0.1.1)
   sidekiq-scheduler (~> 2.1)
   sidekiq-unique-jobs (~> 5.0)
   simple-navigation (~> 4.0)
diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb
index 13722345f..90b48887f 100644
--- a/app/controllers/settings/follower_domains_controller.rb
+++ b/app/controllers/settings/follower_domains_controller.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq-bulk'
+
 class Settings::FollowerDomainsController < ApplicationController
   layout 'admin'
 
@@ -13,8 +15,8 @@ class Settings::FollowerDomainsController < ApplicationController
   def update
     domains = bulk_params[:select] || []
 
-    domains.each do |domain|
-      SoftBlockDomainFollowersWorker.perform_async(current_account.id, domain)
+    SoftBlockDomainFollowersWorker.push_bulk(domains) do |domain|
+      [current_account.id, domain]
     end
 
     redirect_to settings_follower_domains_path, notice: I18n.t('followers.success', count: domains.size)
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 055fda8a9..3b74696d5 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq-bulk'
+
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status
@@ -34,8 +36,10 @@ class FanOutOnWriteService < BaseService
   def deliver_to_followers(status)
     Rails.logger.debug "Delivering status #{status.id} to followers"
 
-    status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_each do |follower|
-      FeedInsertWorker.perform_async(status.id, follower.id)
+    status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers|
+      FeedInsertWorker.push_bulk(followers) do |follower|
+        [status.id, follower.id]
+      end
     end
   end
 
diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb
index 3ea3ad2b8..7bfd002f4 100644
--- a/app/workers/scheduler/subscriptions_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_scheduler.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
+
 require 'sidekiq-scheduler'
+require 'sidekiq-bulk'
 
 class Scheduler::SubscriptionsScheduler
   include Sidekiq::Worker
@@ -7,9 +9,7 @@ class Scheduler::SubscriptionsScheduler
   def perform
     logger.info 'Queueing PuSH re-subscriptions'
 
-    expiring_accounts.pluck(:id).each do |id|
-      Pubsubhubbub::SubscribeWorker.perform_async(id)
-    end
+    Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id))
   end
 
   private
diff --git a/app/workers/soft_block_domain_followers_worker.rb b/app/workers/soft_block_domain_followers_worker.rb
index 2782d05d2..ce76683c5 100644
--- a/app/workers/soft_block_domain_followers_worker.rb
+++ b/app/workers/soft_block_domain_followers_worker.rb
@@ -1,13 +1,16 @@
 # frozen_string_literal: true
 
+require 'sidekiq-bulk'
+
 class SoftBlockDomainFollowersWorker
   include Sidekiq::Worker
 
   sidekiq_options queue: 'pull'
 
   def perform(account_id, domain)
-    Account.find(account_id).followers.where(domain: domain).pluck(:id).each do |follower_id|
-      SoftBlockWorker.perform_async(account_id, follower_id)
+    followers_id = Account.find(account_id).followers.where(domain: domain).pluck(:id)
+    SoftBlockWorker.push_bulk(followers_id) do |follower_id|
+      [account_id, follower_id]
     end
   end
 end