From 47b4cd0fff947a45f149378c9c01a69a8a16985c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:33:54 +0900 Subject: Bump bullet from 6.0.0 to 6.0.1 (#11228) Bumps [bullet](https://github.com/flyerhzm/bullet) from 6.0.0 to 6.0.1. - [Release notes](https://github.com/flyerhzm/bullet/releases) - [Changelog](https://github.com/flyerhzm/bullet/blob/master/CHANGELOG.md) - [Commits](https://github.com/flyerhzm/bullet/compare/6.0.0...6.0.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index 8b0c93b1e..5034c1411 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,7 +106,7 @@ GEM brakeman (4.5.1) browser (2.5.3) builder (3.2.3) - bullet (6.0.0) + bullet (6.0.1) activesupport (>= 3.0.0) uniform_notifier (~> 1.11) bundler-audit (0.6.1) -- cgit From 8c4287c239172e02082c7fec170fffc8b75cab4a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:34:25 +0900 Subject: Bump memory_profiler from 0.9.13 to 0.9.14 (#11227) Bumps [memory_profiler](https://github.com/SamSaffron/memory_profiler) from 0.9.13 to 0.9.14. - [Release notes](https://github.com/SamSaffron/memory_profiler/releases) - [Changelog](https://github.com/SamSaffron/memory_profiler/blob/master/CHANGELOG.md) - [Commits](https://github.com/SamSaffron/memory_profiler/compare/v0.9.13...v0.9.14) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index 5034c1411..8854b5fce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -336,7 +336,7 @@ GEM mimemagic (~> 0.3.2) mario-redis-lock (1.2.1) redis (>= 3.0.5) - memory_profiler (0.9.13) + memory_profiler (0.9.14) method_source (0.9.2) microformats (4.1.0) json (~> 2.1) -- cgit From ffb6da8b782455f3d07750e1faa79e955d72fd54 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:34:51 +0900 Subject: Bump capybara from 3.24.0 to 3.25.0 (#11225) Bumps [capybara](https://github.com/teamcapybara/capybara) from 3.24.0 to 3.25.0. - [Release notes](https://github.com/teamcapybara/capybara/releases) - [Changelog](https://github.com/teamcapybara/capybara/blob/master/History.md) - [Commits](https://github.com/teamcapybara/capybara/compare/3.24.0...3.25.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile b/Gemfile index 4d63cca37..42499cd2c 100644 --- a/Gemfile +++ b/Gemfile @@ -108,7 +108,7 @@ group :production, :test do end group :test do - gem 'capybara', '~> 3.24' + gem 'capybara', '~> 3.25' gem 'climate_control', '~> 0.2' gem 'faker', '~> 1.9' gem 'microformats', '~> 4.1' diff --git a/Gemfile.lock b/Gemfile.lock index 8854b5fce..4a4e86e72 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -129,7 +129,7 @@ GEM sshkit (~> 1.3) capistrano-yarn (2.0.2) capistrano (~> 3.0) - capybara (3.24.0) + capybara (3.25.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -420,7 +420,7 @@ GEM pry (~> 0.10) pry-rails (0.3.9) pry (>= 0.10.4) - public_suffix (3.1.0) + public_suffix (3.1.1) puma (3.12.1) pundit (2.0.1) activesupport (>= 3.0.0) @@ -660,7 +660,7 @@ DEPENDENCIES capistrano-rails (~> 1.4) capistrano-rbenv (~> 2.1) capistrano-yarn (~> 2.0) - capybara (~> 3.24) + capybara (~> 3.25) charlock_holmes (~> 0.7.6) chewy (~> 5.0) cld3 (~> 3.2.4) -- cgit From 13f54f41ade5abfa561c72264481afed5f1cd8a7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2019 22:36:45 +0900 Subject: Bump rubocop from 0.71.0 to 0.72.0 (#11229) Bumps [rubocop](https://github.com/rubocop-hq/rubocop) from 0.71.0 to 0.72.0. - [Release notes](https://github.com/rubocop-hq/rubocop/releases) - [Changelog](https://github.com/rubocop-hq/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop-hq/rubocop/compare/v0.71.0...v0.72.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile b/Gemfile index 42499cd2c..e99a808d4 100644 --- a/Gemfile +++ b/Gemfile @@ -128,7 +128,7 @@ group :development do gem 'letter_opener', '~> 1.7' gem 'letter_opener_web', '~> 1.3' gem 'memory_profiler' - gem 'rubocop', '~> 0.71', require: false + gem 'rubocop', '~> 0.72', require: false gem 'rubocop-rails', '~> 2.0', require: false gem 'brakeman', '~> 4.5', require: false gem 'bundler-audit', '~> 0.6', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 4a4e86e72..c8856ccc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -287,7 +287,7 @@ GEM idn-ruby (0.1.0) ipaddress (0.8.3) iso-639 (0.2.8) - jaro_winkler (1.5.2) + jaro_winkler (1.5.3) jmespath (1.4.0) json (2.1.0) json-ld (3.0.2) @@ -524,7 +524,7 @@ GEM rspec-core (~> 3.0, >= 3.0.0) sidekiq (>= 2.4.0) rspec-support (3.8.0) - rubocop (0.71.0) + rubocop (0.72.0) jaro_winkler (~> 1.5.1) parallel (~> 1.10) parser (>= 2.6) @@ -737,7 +737,7 @@ DEPENDENCIES rqrcode (~> 0.10) rspec-rails (~> 3.8) rspec-sidekiq (~> 3.0) - rubocop (~> 0.71) + rubocop (~> 0.72) rubocop-rails (~> 2.0) sanitize (~> 5.0) sidekiq (~> 5.2) -- cgit From 0d9ffe56fb59e0d1fce91265f44140d874c0bfba Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:34:38 +0200 Subject: Add request pool to improve delivery performance (#10353) * Add request pool to improve delivery performance Fix #7909 * Ensure connection is closed when exception interrupts execution * Remove Timeout#timeout from socket connection * Fix infinite retrial loop on HTTP::ConnectionError * Close sockets on failure, reduce idle time to 90 seconds * Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server * Use a shared pool size, 512 by default, to stay below open file limit * Add some tests * Add more tests * Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds * Use a shared pool that returns preferred connection but re-purposes other ones when needed * Fix wrong connection being returned on subsequent calls within the same thread * Reduce mutex calls on flushes from 2 to 1 and add test for reaping --- Gemfile | 1 + Gemfile.lock | 1 + app/lib/connection_pool/shared_connection_pool.rb | 63 ++++++++++++ app/lib/connection_pool/shared_timed_stack.rb | 95 +++++++++++++++++ app/lib/request.rb | 70 +++++++++---- app/lib/request_pool.rb | 114 +++++++++++++++++++++ app/workers/activitypub/delivery_worker.rb | 15 ++- .../connection_pool/shared_connection_pool_spec.rb | 28 +++++ .../lib/connection_pool/shared_timed_stack_spec.rb | 61 +++++++++++ spec/lib/request_pool_spec.rb | 63 ++++++++++++ 10 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 app/lib/connection_pool/shared_connection_pool.rb create mode 100644 app/lib/connection_pool/shared_timed_stack.rb create mode 100644 app/lib/request_pool.rb create mode 100644 spec/lib/connection_pool/shared_connection_pool_spec.rb create mode 100644 spec/lib/connection_pool/shared_timed_stack_spec.rb create mode 100644 spec/lib/request_pool_spec.rb (limited to 'Gemfile.lock') diff --git a/Gemfile b/Gemfile index e99a808d4..2fcb603c3 100644 --- a/Gemfile +++ b/Gemfile @@ -148,3 +148,4 @@ group :production do end gem 'concurrent-ruby', require: false +gem 'connection_pool', require: false diff --git a/Gemfile.lock b/Gemfile.lock index c8856ccc5..6043a2fa9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -666,6 +666,7 @@ DEPENDENCIES cld3 (~> 3.2.4) climate_control (~> 0.2) concurrent-ruby + connection_pool derailed_benchmarks devise (~> 4.6) devise-two-factor (~> 3.0) diff --git a/app/lib/connection_pool/shared_connection_pool.rb b/app/lib/connection_pool/shared_connection_pool.rb new file mode 100644 index 000000000..2865a4108 --- /dev/null +++ b/app/lib/connection_pool/shared_connection_pool.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'connection_pool' +require_relative './shared_timed_stack' + +class ConnectionPool::SharedConnectionPool < ConnectionPool + def initialize(options = {}, &block) + super(options, &block) + + @available = ConnectionPool::SharedTimedStack.new(@size, &block) + end + + delegate :size, :flush, to: :@available + + def with(preferred_tag, options = {}) + Thread.handle_interrupt(Exception => :never) do + conn = checkout(preferred_tag, options) + + begin + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end + ensure + checkin(preferred_tag) + end + end + end + + def checkout(preferred_tag, options = {}) + if ::Thread.current[key(preferred_tag)] + ::Thread.current[key_count(preferred_tag)] += 1 + ::Thread.current[key(preferred_tag)] + else + ::Thread.current[key_count(preferred_tag)] = 1 + ::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout) + end + end + + def checkin(preferred_tag) + if ::Thread.current[key(preferred_tag)] + if ::Thread.current[key_count(preferred_tag)] == 1 + @available.push(::Thread.current[key(preferred_tag)]) + ::Thread.current[key(preferred_tag)] = nil + else + ::Thread.current[key_count(preferred_tag)] -= 1 + end + else + raise ConnectionPool::Error, 'no connections are checked out' + end + + nil + end + + private + + def key(tag) + :"#{@key}-#{tag}" + end + + def key_count(tag) + :"#{@key_count}-#{tag}" + end +end diff --git a/app/lib/connection_pool/shared_timed_stack.rb b/app/lib/connection_pool/shared_timed_stack.rb new file mode 100644 index 000000000..14a5285c4 --- /dev/null +++ b/app/lib/connection_pool/shared_timed_stack.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +class ConnectionPool::SharedTimedStack + def initialize(max = 0, &block) + @create_block = block + @max = max + @created = 0 + @queue = [] + @tagged_queue = Hash.new { |hash, key| hash[key] = [] } + @mutex = Mutex.new + @resource = ConditionVariable.new + end + + def push(connection) + @mutex.synchronize do + store_connection(connection) + @resource.broadcast + end + end + + alias << push + + def pop(preferred_tag, timeout = 5.0) + deadline = current_time + timeout + + @mutex.synchronize do + loop do + return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty? + + connection = try_create(preferred_tag) + return connection if connection + + to_wait = deadline - current_time + raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0 + + @resource.wait(@mutex, to_wait) + end + end + end + + def empty? + size.zero? + end + + def size + @mutex.synchronize do + @queue.size + end + end + + def flush + @mutex.synchronize do + @queue.delete_if do |connection| + delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME) + + if delete + @tagged_queue[connection.site].delete(connection) + connection.close + @created -= 1 + end + + delete + end + end + end + + private + + def try_create(preferred_tag) + if @created == @max && !@queue.empty? + throw_away_connection = @queue.pop + @tagged_queue[throw_away_connection.site].delete(throw_away_connection) + @create_block.call(preferred_tag) + elsif @created != @max + connection = @create_block.call(preferred_tag) + @created += 1 + connection + end + end + + def fetch_preferred_connection(preferred_tag) + connection = @tagged_queue[preferred_tag].pop + @queue.delete(connection) + connection + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def store_connection(connection) + @tagged_queue[connection.site].push(connection) + @queue.push(connection) + end +end diff --git a/app/lib/request.rb b/app/lib/request.rb index e555ae6a1..af49d6c77 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -17,15 +17,21 @@ end class Request REQUEST_TARGET = '(request-target)' + # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening + # and 5s timeout on the TLS handshake, meaning the worst case should take + # about 15s in total + TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze + include RoutingHelper def initialize(verb, url, **options) raise ArgumentError if url.blank? - @verb = verb - @url = Addressable::URI.parse(url).normalize - @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) - @headers = {} + @verb = verb + @url = Addressable::URI.parse(url).normalize + @http_client = options.delete(:http_client) + @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) + @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -50,15 +56,24 @@ class Request def perform begin - response = http_client.headers(headers).public_send(@verb, @url.to_s, @options) + response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers)) rescue => e raise e.class, "#{e.message} on #{@url}", e.backtrace[0] end begin - yield response.extend(ClientLimit) if block_given? + response = response.extend(ClientLimit) + + # If we are using a persistent connection, we have to + # read every response to be able to move forward at all. + # However, simply calling #to_s or #flush may not be safe, + # as the response body, if malicious, could be too big + # for our memory. So we use the #body_with_limit method + response.body_with_limit if http_client.persistent? + + yield response if block_given? ensure - http_client.close + http_client.close unless http_client.persistent? end end @@ -76,6 +91,10 @@ class Request %w(http https).include?(parsed_url.scheme) && parsed_url.host.present? end + + def http_client + HTTP.use(:auto_inflate).timeout(:per_operation, TIMEOUT.dup).follow(max_hops: 2) + end end private @@ -116,16 +135,8 @@ class Request end end - def timeout - # We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening - # and 5s timeout on the TLS handshake, meaning the worst case should take - # about 16s in total - - { connect: 5, read: 10, write: 10 } - end - def http_client - @http_client ||= HTTP.use(:auto_inflate).timeout(:per_operation, timeout).follow(max_hops: 2) + @http_client ||= Request.http_client end def use_proxy? @@ -169,20 +180,41 @@ class Request return super(host, *args) if thru_hidden_service?(host) outer_e = nil + port = args.first Resolv::DNS.open do |dns| dns.timeouts = 5 addresses = dns.getaddresses(host).take(2) - time_slot = 10.0 / addresses.size addresses.each do |address| begin raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) - ::Timeout.timeout(time_slot, HTTP::TimeoutError) do - return super(address.to_s, *args) + sock = ::Socket.new(::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) + sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) + + sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) + + begin + sock.connect_nonblock(sockaddr) + rescue IO::WaitWritable + if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) + begin + sock.connect_nonblock(sockaddr) + rescue Errno::EISCONN + # Yippee! + rescue + sock.close + raise + end + else + sock.close + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + end end + + return sock rescue => e outer_e = e end diff --git a/app/lib/request_pool.rb b/app/lib/request_pool.rb new file mode 100644 index 000000000..e5899a79a --- /dev/null +++ b/app/lib/request_pool.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative './connection_pool/shared_connection_pool' + +class RequestPool + def self.current + @current ||= RequestPool.new + end + + class Reaper + attr_reader :pool, :frequency + + def initialize(pool, frequency) + @pool = pool + @frequency = frequency + end + + def run + return unless frequency&.positive? + + Thread.new(frequency, pool) do |t, p| + loop do + sleep t + p.flush + end + end + end + end + + MAX_IDLE_TIME = 30 + WAIT_TIMEOUT = 5 + MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i + + class Connection + attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh + + def initialize(site) + @site = site + @http_client = http_client + @last_used_at = nil + @created_at = current_time + @dead = false + @fresh = true + end + + def use + @last_used_at = current_time + @in_use = true + + retries = 0 + + begin + yield @http_client + rescue HTTP::ConnectionError + # It's possible the connection was closed, so let's + # try re-opening it once + + close + + if @fresh || retries.positive? + raise + else + @http_client = http_client + retries += 1 + retry + end + rescue StandardError + # If this connection raises errors of any kind, it's + # better if it gets reaped as soon as possible + + close + @dead = true + raise + end + ensure + @fresh = false + @in_use = false + end + + def seconds_idle + current_time - (@last_used_at || @created_at) + end + + def close + @http_client.close + end + + private + + def http_client + Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME) + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + end + + def initialize + @pool = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) } + @reaper = Reaper.new(self, 30) + @reaper.run + end + + def with(site, &block) + @pool.with(site) do |connection| + ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do + connection.use(&block) + end + end + end + + delegate :size, :flush, to: :@pool +end diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 5e4c391f0..79f1e8153 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -17,6 +17,7 @@ class ActivityPub::DeliveryWorker @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url + @host = Addressable::URI.parse(inbox_url).normalized_site perform_request @@ -28,16 +29,18 @@ class ActivityPub::DeliveryWorker private - def build_request - request = Request.new(:post, @inbox_url, body: @json) + def build_request(http_client) + request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) request.add_headers(HEADERS) end def perform_request light = Stoplight(@inbox_url) do - build_request.perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + request_pool.with(@host) do |http_client| + build_request(http_client).perform do |response| + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + end end end @@ -57,4 +60,8 @@ class ActivityPub::DeliveryWorker def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end + + def request_pool + RequestPool.current + end end diff --git a/spec/lib/connection_pool/shared_connection_pool_spec.rb b/spec/lib/connection_pool/shared_connection_pool_spec.rb new file mode 100644 index 000000000..114464558 --- /dev/null +++ b/spec/lib/connection_pool/shared_connection_pool_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedConnectionPool do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } } + + describe '#with' do + it 'runs a block with a connection' do + block_run = false + + subject.with('foo') do |connection| + expect(connection).to be_a MiniConnection + block_run = true + end + + expect(block_run).to be true + end + end +end diff --git a/spec/lib/connection_pool/shared_timed_stack_spec.rb b/spec/lib/connection_pool/shared_timed_stack_spec.rb new file mode 100644 index 000000000..f680c5966 --- /dev/null +++ b/spec/lib/connection_pool/shared_timed_stack_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedTimedStack do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(5) { |site| MiniConnection.new(site) } } + + describe '#push' do + it 'keeps the connection in the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.size).to eq 1 + end + end + + describe '#pop' do + it 'returns a connection' do + expect(subject.pop('foo')).to be_a MiniConnection + end + + it 'returns the same connection that was pushed in' do + connection = MiniConnection.new('foo') + subject.push(connection) + expect(subject.pop('foo')).to be connection + end + + it 'does not create more than maximum amount of connections' do + expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error + end + + it 'repurposes a connection for a different site when maximum amount is reached' do + 5.times { subject.push(MiniConnection.new('foo')) } + expect(subject.pop('bar')).to be_a MiniConnection + end + end + + describe '#empty?' do + it 'returns true when no connections on the stack' do + expect(subject.empty?).to be true + end + + it 'returns false when there are connections on the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.empty?).to be false + end + end + + describe '#size' do + it 'returns the number of connections on the stack' do + 2.times { subject.push(MiniConnection.new('foo')) } + expect(subject.size).to eq 2 + end + end +end diff --git a/spec/lib/request_pool_spec.rb b/spec/lib/request_pool_spec.rb new file mode 100644 index 000000000..4a144d7c7 --- /dev/null +++ b/spec/lib/request_pool_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe RequestPool do + subject { described_class.new } + + describe '#with' do + it 'returns a HTTP client for a host' do + subject.with('http://example.com') do |http_client| + expect(http_client).to be_a HTTP::Client + end + end + + it 'returns the same instance of HTTP client within the same thread for the same host' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.com') { |http_client| expect(http_client).to be test_client } + end + + it 'returns different HTTP clients for different hosts' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client } + end + + it 'grows to the number of threads accessing it' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject + + threads = 20.times.map do |i| + Thread.new do + 20.times do + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + end + end + end + + threads.map(&:join) + + expect(subject.size).to be > 1 + end + + it 'closes idle connections' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + + expect(subject.size).to eq 1 + sleep RequestPool::MAX_IDLE_TIME + 30 + 1 + expect(subject.size).to eq 0 + end + end +end -- cgit