Change auto-deletion throttling constants to better scale with server size (#23320)

1 files changed, 8 insertions, 16 deletions

diff --git a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
index 09e0b37f0..f237f1dc9 100644
--- a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
+++ b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb
@@ -7,7 +7,7 @@ class Scheduler::AccountsStatusesCleanupScheduler
   # This limit is mostly to be nice to the fediverse at large and not
   # generate too much traffic.
   # This also helps limiting the running time of the scheduler itself.
-  MAX_BUDGET         = 50
+  MAX_BUDGET         = 150
 
   # This is an attempt to spread the load across instances, as various
   # accounts are likely to have various followers.
@@ -15,28 +15,22 @@ class Scheduler::AccountsStatusesCleanupScheduler
 
   # This is an attempt to limit the workload generated by status removal
   # jobs to something the particular instance can handle.
-  PER_THREAD_BUDGET  = 5
+  PER_THREAD_BUDGET  = 6
 
   # Those avoid loading an instance that is already under load
-  MAX_DEFAULT_SIZE    = 2
+  MAX_DEFAULT_SIZE    = 200
   MAX_DEFAULT_LATENCY = 5
-  MAX_PUSH_SIZE       = 5
+  MAX_PUSH_SIZE       = 500
   MAX_PUSH_LATENCY    = 10
+
   # 'pull' queue has lower priority jobs, and it's unlikely that pushing
   # deletes would cause much issues with this queue if it didn't cause issues
   # with default and push. Yet, do not enqueue deletes if the instance is
   # lagging behind too much.
-  MAX_PULL_SIZE       = 500
-  MAX_PULL_LATENCY    = 300
-
-  # This is less of an issue in general, but deleting old statuses is likely
-  # to cause delivery errors, and thus increase the number of jobs to be retried.
-  # This doesn't directly translate to load, but connection errors and a high
-  # number of dead instances may lead to this spiraling out of control if
-  # unchecked.
-  MAX_RETRY_SIZE = 50_000
+  MAX_PULL_SIZE       = 10_000
+  MAX_PULL_LATENCY    = 5.minutes.to_i
 
-  sidekiq_options retry: 0, lock: :until_executed
+  sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
 
   def perform
     return if under_load?
@@ -73,8 +67,6 @@ class Scheduler::AccountsStatusesCleanupScheduler
   end
 
   def under_load?
-    return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE
-
     queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY)
   end