mastodon/app/workers/scheduler/indexing_scheduler.rb
Vyr Cossont 7646ad8a2b
IndexingScheduler: fetch and import in batches (#24285)
Co-authored-by: Claire <claire.github-309c@sitedethib.com>
2023-03-31 14:38:47 +02:00

33 lines
730 B
Ruby

# frozen_string_literal: true
class Scheduler::IndexingScheduler
include Sidekiq::Worker
include Redisable
sidekiq_options retry: 0
IMPORT_BATCH_SIZE = 1000
SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
def perform
return unless Chewy.enabled?
indexes.each do |type|
with_redis do |redis|
redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE) do |ids|
redis.pipelined do
ids.each_slice(IMPORT_BATCH_SIZE) do |slice_ids|
type.import!(slice_ids)
redis.srem("chewy:queue:#{type.name}", slice_ids)
end
end
end
end
end
end
def indexes
[AccountsIndex, TagsIndex, StatusesIndex]
end
end