Separate background jobs into different queues. ATTENTION: new queue "pull"

must be added to the Sidekiq invokation in your systemd file

The pull queue will handle link crawling, thread resolving, and OStatus
processing. Such tasks are more likely to hang for a longer time (due to
network requests) so it is more sensible to not make the "in-house" tasks
wait for them.

13 files changed, 16 insertions, 10 deletions

diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb
index f1d6869cc..1f2db3061 100644
--- a/app/workers/after_remote_follow_request_worker.rb
+++ b/app/workers/after_remote_follow_request_worker.rb
@@ -3,7 +3,7 @@
 class AfterRemoteFollowRequestWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: 5
+  sidekiq_options queue: 'pull', retry: 5
 
   def perform(follow_request_id)
     follow_request  = FollowRequest.find(follow_request_id)
diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb
index 0d04456a9..bdd2c2a91 100644
--- a/app/workers/after_remote_follow_worker.rb
+++ b/app/workers/after_remote_follow_worker.rb
@@ -3,7 +3,7 @@
 class AfterRemoteFollowWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: 5
+  sidekiq_options queue: 'pull', retry: 5
 
   def perform(follow_id)
     follow          = Follow.find(follow_id)
diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb
index a3ae2a85a..7cf29fb53 100644
--- a/app/workers/import_worker.rb
+++ b/app/workers/import_worker.rb
@@ -5,7 +5,7 @@ require 'csv'
 class ImportWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: false
+  sidekiq_options queue: 'pull', retry: false
 
   def perform(import_id)
     import = Import.find(import_id)
diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb
index af3394b8b..834b0088b 100644
--- a/app/workers/link_crawl_worker.rb
+++ b/app/workers/link_crawl_worker.rb
@@ -3,7 +3,7 @@
 class LinkCrawlWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: false
+  sidekiq_options queue: 'pull', retry: false
 
   def perform(status_id)
     FetchLinkCardService.new.call(Status.find(status_id))
diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb
index 0f288f43f..d745cb99c 100644
--- a/app/workers/merge_worker.rb
+++ b/app/workers/merge_worker.rb
@@ -3,6 +3,8 @@
 class MergeWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull'
+
   def perform(from_account_id, into_account_id)
     FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id))
   end
diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb
index 1a2faefd8..da1d6ab45 100644
--- a/app/workers/notification_worker.rb
+++ b/app/workers/notification_worker.rb
@@ -3,7 +3,7 @@
 class NotificationWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: 5
+  sidekiq_options queue: 'push', retry: 5
 
   def perform(xml, source_account_id, target_account_id)
     SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id))
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc..4a467d924 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -3,7 +3,7 @@
 class ProcessingWorker
   include Sidekiq::Worker
 
-  sidekiq_options backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true
 
   def perform(account_id, body)
     ProcessFeedService.new.call(body, Account.find(account_id))
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 3aece0ba2..289b63d84 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,6 +3,8 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull', backtrace: true
+
   def perform(account_id, timeline_type)
     PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
   end
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index fc95ce47f..2888b574b 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -3,7 +3,7 @@
 class SalmonWorker
   include Sidekiq::Worker
 
-  sidekiq_options backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true
 
   def perform(account_id, body)
     ProcessInteractionService.new.call(body, Account.find(account_id))
diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb
index 593edd032..38287e8e6 100644
--- a/app/workers/thread_resolve_worker.rb
+++ b/app/workers/thread_resolve_worker.rb
@@ -3,7 +3,7 @@
 class ThreadResolveWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: false
+  sidekiq_options queue: 'pull', retry: false
 
   def perform(child_status_id, parent_url)
     child_status  = Status.find(child_status_id)
diff --git a/app/workers/unmerge_worker.rb b/app/workers/unmerge_worker.rb
index dbf7243de..ea6aacebf 100644
--- a/app/workers/unmerge_worker.rb
+++ b/app/workers/unmerge_worker.rb
@@ -3,6 +3,8 @@
 class UnmergeWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull'
+
   def perform(from_account_id, into_account_id)
     FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id))
   end
diff --git a/docker-compose.yml b/docker-compose.yml
index 68c8ef960..d6ba66dde 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -33,7 +33,7 @@ services:
     restart: always
     build: .
     env_file: .env.production
-    command: bundle exec sidekiq -q default -q mailers -q push
+    command: bundle exec sidekiq -q default -q mailers -q pull -q push
     depends_on:
       - db
       - redis
diff --git a/docs/Running-Mastodon/Production-guide.md b/docs/Running-Mastodon/Production-guide.md
index f0dd7bd2b..469fefa94 100644
--- a/docs/Running-Mastodon/Production-guide.md
+++ b/docs/Running-Mastodon/Production-guide.md
@@ -180,7 +180,7 @@ User=mastodon
 WorkingDirectory=/home/mastodon/live
 Environment="RAILS_ENV=production"
 Environment="DB_POOL=5"
-ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q push
+ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push
 TimeoutSec=15
 Restart=always