From 3eedad27375721aaaa6e8397124989e4ead771a9 Mon Sep 17 00:00:00 2001 From: takayamaki Date: Sun, 4 Jun 2017 07:11:15 +0900 Subject: [PATCH] change sidekiq queueing to bulk push (#3536) --- Gemfile | 1 + Gemfile.lock | 4 ++++ app/controllers/settings/follower_domains_controller.rb | 6 ++++-- app/services/fan_out_on_write_service.rb | 8 ++++++-- app/workers/scheduler/subscriptions_scheduler.rb | 6 +++--- app/workers/soft_block_domain_followers_worker.rb | 7 +++++-- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 835a6996a8..d5fbab7485 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,7 @@ gem 'sanitize', '~> 4.4' gem 'sidekiq', '~> 5.0' gem 'sidekiq-scheduler', '~> 2.1' gem 'sidekiq-unique-jobs', '~> 5.0' +gem 'sidekiq-bulk', '~>0.1.1' gem 'simple-navigation', '~> 4.0' gem 'simple_form', '~> 3.4' gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie' diff --git a/Gemfile.lock b/Gemfile.lock index c7ea6d79ed..ee86bfafd1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -402,6 +402,9 @@ GEM connection_pool (~> 2.2, >= 2.2.0) rack-protection (>= 1.5.0) redis (~> 3.3, >= 3.3.3) + sidekiq-bulk (0.1.1) + activesupport + sidekiq sidekiq-scheduler (2.1.4) redis (~> 3) rufus-scheduler (~> 3.2) @@ -543,6 +546,7 @@ DEPENDENCIES sanitize (~> 4.4) scss_lint (~> 0.53) sidekiq (~> 5.0) + sidekiq-bulk (~> 0.1.1) sidekiq-scheduler (~> 2.1) sidekiq-unique-jobs (~> 5.0) simple-navigation (~> 4.0) diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb index 13722345fd..90b48887fa 100644 --- a/app/controllers/settings/follower_domains_controller.rb +++ b/app/controllers/settings/follower_domains_controller.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'sidekiq-bulk' + class Settings::FollowerDomainsController < ApplicationController layout 'admin' @@ -13,8 +15,8 @@ class Settings::FollowerDomainsController < ApplicationController def update domains = bulk_params[:select] || [] - domains.each do |domain| - SoftBlockDomainFollowersWorker.perform_async(current_account.id, domain) + SoftBlockDomainFollowersWorker.push_bulk(domains) do |domain| + [current_account.id, domain] end redirect_to settings_follower_domains_path, notice: I18n.t('followers.success', count: domains.size) diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 055fda8a9b..3b74696d5b 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'sidekiq-bulk' + class FanOutOnWriteService < BaseService # Push a status into home and mentions feeds # @param [Status] status @@ -34,8 +36,10 @@ class FanOutOnWriteService < BaseService def deliver_to_followers(status) Rails.logger.debug "Delivering status #{status.id} to followers" - status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_each do |follower| - FeedInsertWorker.perform_async(status.id, follower.id) + status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers| + FeedInsertWorker.push_bulk(followers) do |follower| + [status.id, follower.id] + end end end diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb index 3ea3ad2b89..7bfd002f47 100644 --- a/app/workers/scheduler/subscriptions_scheduler.rb +++ b/app/workers/scheduler/subscriptions_scheduler.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true + require 'sidekiq-scheduler' +require 'sidekiq-bulk' class Scheduler::SubscriptionsScheduler include Sidekiq::Worker @@ -7,9 +9,7 @@ class Scheduler::SubscriptionsScheduler def perform logger.info 'Queueing PuSH re-subscriptions' - expiring_accounts.pluck(:id).each do |id| - Pubsubhubbub::SubscribeWorker.perform_async(id) - end + Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id)) end private diff --git a/app/workers/soft_block_domain_followers_worker.rb b/app/workers/soft_block_domain_followers_worker.rb index 2782d05d27..ce76683c50 100644 --- a/app/workers/soft_block_domain_followers_worker.rb +++ b/app/workers/soft_block_domain_followers_worker.rb @@ -1,13 +1,16 @@ # frozen_string_literal: true +require 'sidekiq-bulk' + class SoftBlockDomainFollowersWorker include Sidekiq::Worker sidekiq_options queue: 'pull' def perform(account_id, domain) - Account.find(account_id).followers.where(domain: domain).pluck(:id).each do |follower_id| - SoftBlockWorker.perform_async(account_id, follower_id) + followers_id = Account.find(account_id).followers.where(domain: domain).pluck(:id) + SoftBlockWorker.push_bulk(followers_id) do |follower_id| + [account_id, follower_id] end end end