From d890abfcab076f5df4eafb75397435d32c641e20 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Tue, 9 Jun 2020 17:26:58 +0900 Subject: [PATCH] Fix performance of follow import (#13836) --- app/helpers/webfinger_helper.rb | 19 +++++++++++++++++++ app/services/import_service.rb | 4 +++- app/services/resolve_account_service.rb | 2 ++ app/workers/import/relationship_worker.rb | 21 ++++++++++++++++++++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/app/helpers/webfinger_helper.rb b/app/helpers/webfinger_helper.rb index 70c4932100..ab7ca46981 100644 --- a/app/helpers/webfinger_helper.rb +++ b/app/helpers/webfinger_helper.rb @@ -1,5 +1,16 @@ # frozen_string_literal: true +# Monkey-patch on monkey-patch. +# Because it conflicts with the request.rb patch. +class HTTP::Timeout::PerOperationOriginal < HTTP::Timeout::PerOperation + def connect(socket_class, host, port, nodelay = false) + ::Timeout.timeout(@connect_timeout, HTTP::TimeoutError) do + @socket = socket_class.open(host, port) + @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay + end + end +end + module WebfingerHelper def webfinger!(uri) hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri) @@ -12,6 +23,14 @@ module WebfingerHelper headers: { 'User-Agent': Mastodon::Version.user_agent, }, + + timeout_class: HTTP::Timeout::PerOperationOriginal, + + timeout_options: { + write_timeout: 10, + connect_timeout: 5, + read_timeout: 10, + }, } Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger diff --git a/app/services/import_service.rb b/app/services/import_service.rb index c0d741d570..4cad93767a 100644 --- a/app/services/import_service.rb +++ b/app/services/import_service.rb @@ -81,7 +81,9 @@ class ImportService < BaseService end end - Import::RelationshipWorker.push_bulk(items) do |acct, extra| + head_items = items.uniq { |acct, _| acct.split('@')[1] } + tail_items = items - head_items + Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra| [@account.id, acct, action, extra] end end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 17ace100cb..ba77552c6c 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -112,6 +112,8 @@ class ResolveAccountService < BaseService end def webfinger_update_due? + return false if @options[:check_delivery_availability] && !DeliveryFailureTracker.available?(@domain) + @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?) end diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb index 616da6da92..4a455f3aeb 100644 --- a/app/workers/import/relationship_worker.rb +++ b/app/workers/import/relationship_worker.rb @@ -7,7 +7,8 @@ class Import::RelationshipWorker def perform(account_id, target_account_uri, relationship, options = {}) from_account = Account.find(account_id) - target_account = ResolveAccountService.new.call(target_account_uri) + target_domain = domain(target_account_uri) + target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) } options.symbolize_keys! return if target_account.nil? @@ -29,4 +30,22 @@ class Import::RelationshipWorker rescue ActiveRecord::RecordNotFound true end + + def domain(uri) + domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1] + TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain) + end + + def stoplight_wrap_request(domain, &block) + if domain.present? + Stoplight("source:#{domain}", &block) + .with_fallback { nil } + .with_threshold(1) + .with_cool_off_time(5.minutes.seconds) + .with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) } + .run + else + block.call + end + end end