mastodon/app/lib/request.rb

297 lines
8.2 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
require 'ipaddr'
require 'socket'
require 'resolv'
# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
# around the Socket#open method, since we use our own timeout blocks inside
# that method
class HTTP::Timeout::PerOperation
def connect(socket_class, host, port, nodelay = false)
@socket = socket_class.open(host, port)
@socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
end
end
class Request
REQUEST_TARGET = '(request-target)'
# We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
# and 5s timeout on the TLS handshake, meaning the worst case should take
# about 15s in total
TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
include RoutingHelper
2017-12-06 11:41:57 +01:00
def initialize(verb, url, **options)
raise ArgumentError if url.blank?
@verb = verb
@url = Addressable::URI.parse(url).normalize
@http_client = options.delete(:http_client)
@allow_local = options.delete(:allow_local)
@options = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
@options = @options.merge(proxy_url) if use_proxy?
@headers = {}
raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
set_common_headers!
set_digest! if options.key?(:body)
end
def on_behalf_of(actor, sign_with: nil)
raise ArgumentError, 'actor must not be nil' if actor.nil?
@actor = actor
@keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @actor.keypair
self
end
def add_headers(new_headers)
@headers.merge!(new_headers)
self
end
def perform
begin
response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
rescue => e
Suppress backtrace when delivering toots (#12798) This is to suppress irrelevant backtrace from errors raised when delivering toots to remote servers. The errors are usually out of control by the local server and backtraces don't provide much information. This is similar to https://github.com/tootsuite/mastodon/pull/5174 and shortens backtraces like below: ``` WARN: Mastodon::UnexpectedResponseError: https://example.com/inbox returned code 523 WARN: app/workers/activitypub/delivery_worker.rb:48:in `block (3 levels) in perform_request' app/lib/request.rb:75:in `perform' app/workers/activitypub/delivery_worker.rb:47:in `block (2 levels) in perform_request' app/lib/request_pool.rb:53:in `use' app/lib/request_pool.rb:108:in `block (2 levels) in with' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/notifications.rb:170:in `instrument' app/lib/request_pool.rb:107:in `block in with' app/lib/connection_pool/shared_connection_pool.rb:21:in `block (2 levels) in with' app/lib/connection_pool/shared_connection_pool.rb:20:in `handle_interrupt' app/lib/connection_pool/shared_connection_pool.rb:20:in `block in with' app/lib/connection_pool/shared_connection_pool.rb:16:in `handle_interrupt' app/lib/connection_pool/shared_connection_pool.rb:16:in `with' app/lib/request_pool.rb:106:in `with' app/workers/activitypub/delivery_worker.rb:46:in `block in perform_request' vendor/bundle/ruby/2.7.0/gems/stoplight-2.2.0/lib/stoplight/light/runnable.rb:51:in `run_code' vendor/bundle/ruby/2.7.0/gems/stoplight-2.2.0/lib/stoplight/light/runnable.rb:42:in `run_yellow' vendor/bundle/ruby/2.7.0/gems/stoplight-2.2.0/lib/stoplight/light/runnable.rb:24:in `run' app/workers/activitypub/delivery_worker.rb:57:in `perform_request' app/workers/activitypub/delivery_worker.rb:25:in `perform' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:192:in `execute_job' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:165:in `block (2 levels) in process' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:128:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/nsa-0.2.7/lib/nsa/collectors/sidekiq.rb:31:in `block in call' vendor/bundle/ruby/2.7.0/gems/nsa-0.2.7/lib/nsa/statsd/publisher.rb:27:in `statsd_time' vendor/bundle/ruby/2.7.0/gems/nsa-0.2.7/lib/nsa/collectors/sidekiq.rb:30:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' app/lib/sidekiq_error_handler.rb:5:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/scout_apm-2.3.0.pre3/lib/scout_apm/background_job_integrations/sidekiq.rb:69:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/sidekiq-unique-jobs-6.0.18/lib/sidekiq_unique_jobs/server/middleware.rb:29:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:133:in `invoke' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:164:in `block in process' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:137:in `block (6 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/job_retry.rb:109:in `local' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:136:in `block (5 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/rails.rb:43:in `block in call' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/execution_wrapper.rb:87:in `wrap' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/reloader.rb:73:in `block in wrap' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/execution_wrapper.rb:87:in `wrap' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/reloader.rb:72:in `wrap' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/rails.rb:42:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:132:in `block (4 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:250:in `stats' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:127:in `block (3 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/job_logger.rb:8:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:126:in `block (2 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/job_retry.rb:74:in `global' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:125:in `block in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/logging.rb:48:in `with_context' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/logging.rb:42:in `with_job_hash_context' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:124:in `dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:163:in `process' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:83:in `process_one' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:71:in `run' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/util.rb:16:in `watchdog' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/util.rb:25:in `block in safe_thread' ``` ``` WARN: Stoplight::Error::RedLight: https://example.com/inbox WARN: vendor/bundle/ruby/2.7.0/gems/stoplight-2.2.0/lib/stoplight/light/runnable.rb:46:in `run_red' vendor/bundle/ruby/2.7.0/gems/stoplight-2.2.0/lib/stoplight/light/runnable.rb:25:in `run' app/workers/activitypub/delivery_worker.rb:57:in `perform_request' app/workers/activitypub/delivery_worker.rb:25:in `perform' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:192:in `execute_job' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:165:in `block (2 levels) in process' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:128:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/nsa-0.2.7/lib/nsa/collectors/sidekiq.rb:31:in `block in call' vendor/bundle/ruby/2.7.0/gems/nsa-0.2.7/lib/nsa/statsd/publisher.rb:27:in `statsd_time' vendor/bundle/ruby/2.7.0/gems/nsa-0.2.7/lib/nsa/collectors/sidekiq.rb:30:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' app/lib/sidekiq_error_handler.rb:5:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/scout_apm-2.3.0.pre3/lib/scout_apm/background_job_integrations/sidekiq.rb:69:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/sidekiq-unique-jobs-6.0.18/lib/sidekiq_unique_jobs/server/middleware.rb:29:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:130:in `block in invoke' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:133:in `invoke' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:164:in `block in process' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:137:in `block (6 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/job_retry.rb:109:in `local' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:136:in `block (5 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/rails.rb:43:in `block in call' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/execution_wrapper.rb:87:in `wrap' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/reloader.rb:73:in `block in wrap' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/execution_wrapper.rb:87:in `wrap' vendor/bundle/ruby/2.7.0/gems/activesupport-5.2.4.1/lib/active_support/reloader.rb:72:in `wrap' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/rails.rb:42:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:132:in `block (4 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:250:in `stats' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:127:in `block (3 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/job_logger.rb:8:in `call' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:126:in `block (2 levels) in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/job_retry.rb:74:in `global' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:125:in `block in dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/logging.rb:48:in `with_context' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/logging.rb:42:in `with_job_hash_context' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:124:in `dispatch' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:163:in `process' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:83:in `process_one' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:71:in `run' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/util.rb:16:in `watchdog' vendor/bundle/ruby/2.7.0/gems/sidekiq-5.2.7/lib/sidekiq/util.rb:25:in `block in safe_thread' ```
2020-01-11 02:15:03 +01:00
raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
end
begin
# If we are using a persistent connection, we have to
# read every response to be able to move forward at all.
# However, simply calling #to_s or #flush may not be safe,
# as the response body, if malicious, could be too big
# for our memory. So we use the #body_with_limit method
response.body_with_limit if http_client.persistent?
yield response if block_given?
ensure
http_client.close unless http_client.persistent?
end
end
def headers
(@actor ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET)
end
class << self
def valid_url?(url)
begin
parsed_url = Addressable::URI.parse(url)
rescue Addressable::URI::InvalidURIError
return false
end
%w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
end
def http_client
HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
end
end
private
def set_common_headers!
@headers[REQUEST_TARGET] = "#{@verb} #{@url.path}"
@headers['User-Agent'] = Mastodon::Version.user_agent
@headers['Host'] = @url.host
@headers['Date'] = Time.now.utc.httpdate
@headers['Accept-Encoding'] = 'gzip' if @verb != :head
end
def set_digest!
@headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
end
def signature
algorithm = 'rsa-sha256'
signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
"keyId=\"#{key_id}\",algorithm=\"#{algorithm}\",headers=\"#{signed_headers.keys.join(' ').downcase}\",signature=\"#{signature}\""
end
def signed_string
signed_headers.map { |key, value| "#{key.downcase}: #{value}" }.join("\n")
end
def signed_headers
@headers.without('User-Agent', 'Accept-Encoding')
end
def key_id
ActivityPub::TagManager.instance.key_uri_for(@actor)
end
def http_client
@http_client ||= Request.http_client
end
def use_proxy?
proxy_url.present?
end
def proxy_url
if hidden_service? && Rails.configuration.x.http_client_hidden_proxy.present?
Rails.configuration.x.http_client_hidden_proxy
else
Rails.configuration.x.http_client_proxy
end
end
def block_hidden_service?
!Rails.configuration.x.access_to_hidden_service && hidden_service?
end
def hidden_service?
/\.(onion|i2p)$/.match?(@url.host)
end
module ClientLimit
def truncated_body(limit = 1.megabyte)
if charset.nil?
encoding = Encoding::BINARY
else
begin
encoding = Encoding.find(charset)
rescue ArgumentError
encoding = Encoding::BINARY
end
end
contents = String.new(encoding: encoding)
while (chunk = readpartial)
contents << chunk
chunk.clear
break if contents.bytesize > limit
end
contents
end
def body_with_limit(limit = 1.megabyte)
raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
contents = truncated_body(limit)
raise Mastodon::LengthValidationError if contents.bytesize > limit
contents
end
end
if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production?
abort 'HTTP::Response#body_with_limit is already defined, the monkey patch will not be applied'
else
class ::HTTP::Response
include Request::ClientLimit
end
end
class Socket < TCPSocket
class << self
def open(host, *args)
outer_e = nil
port = args.first
addresses = []
begin
addresses = [IPAddr.new(host)]
rescue IPAddr::InvalidAddressError
Resolv::DNS.open do |dns|
dns.timeouts = 5
addresses = dns.getaddresses(host)
addresses = addresses.filter { |addr| addr.is_a?(Resolv::IPv6) }.take(2) + addresses.filter { |addr| !addr.is_a?(Resolv::IPv6) }.take(2)
end
end
socks = []
addr_by_socket = {}
addresses.each do |address|
check_private_address(address, host)
sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
sock.connect_nonblock(sockaddr)
# If that hasn't raised an exception, we somehow managed to connect
# immediately, close pending sockets and return immediately
socks.each(&:close)
return sock
rescue IO::WaitWritable
socks << sock
addr_by_socket[sock] = sockaddr
rescue => e
outer_e = e
end
until socks.empty?
_, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
if available_socks.nil?
socks.each(&:close)
raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
end
available_socks.each do |sock|
socks.delete(sock)
begin
sock.connect_nonblock(addr_by_socket[sock])
rescue Errno::EISCONN
# Do nothing
rescue => e
sock.close
outer_e = e
next
end
socks.each(&:close)
return sock
end
end
if outer_e
raise outer_e
else
raise SocketError, "No address for #{host}"
end
end
alias new open
def check_private_address(address, host)
addr = IPAddr.new(address.to_s)
return if private_address_exceptions.any? { |range| range.include?(addr) }
raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr)
end
def private_address_exceptions
@private_address_exceptions = (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(',').map { |addr| IPAddr.new(addr) }
end
end
end
class ProxySocket < Socket
class << self
def check_private_address(_address, _host)
# Accept connections to private addresses as HTTP proxies will usually
# be on local addresses
nil
end
end
end
private_constant :ClientLimit, :Socket, :ProxySocket
end