diff options
author | Starfall <us@starfall.systems> | 2023-04-14 19:22:47 -0500 |
---|---|---|
committer | Starfall <us@starfall.systems> | 2023-04-14 19:22:47 -0500 |
commit | 4fe1689de43f4404eb9530fcfbcbfb26d6c1c13a (patch) | |
tree | 6811b845bb7f4966b10dcefa3dea404246f161c7 /app/workers/scheduler/indexing_scheduler.rb | |
parent | 65c1e53a32cabcdbb7bca57002bb0f6acdebe07e (diff) | |
parent | bed63f6dae0879ac840066b031229e0d139089cd (diff) |
Diffstat (limited to 'app/workers/scheduler/indexing_scheduler.rb')
-rw-r--r-- | app/workers/scheduler/indexing_scheduler.rb | 14 |
1 files changed, 8 insertions, 6 deletions
diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index c42396629..d622f5586 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -6,17 +6,19 @@ class Scheduler::IndexingScheduler sidekiq_options retry: 0 + IMPORT_BATCH_SIZE = 1000 + SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE + def perform return unless Chewy.enabled? indexes.each do |type| with_redis do |redis| - ids = redis.smembers("chewy:queue:#{type.name}") - - type.import!(ids) - - redis.pipelined do |pipeline| - ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } + redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| + type.import!(ids) + redis.pipelined do |pipeline| + pipeline.srem("chewy:queue:#{type.name}", ids) + end end end end |