From 0d9ffe56fb59e0d1fce91265f44140d874c0bfba Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:34:38 +0200 Subject: Add request pool to improve delivery performance (#10353) * Add request pool to improve delivery performance Fix #7909 * Ensure connection is closed when exception interrupts execution * Remove Timeout#timeout from socket connection * Fix infinite retrial loop on HTTP::ConnectionError * Close sockets on failure, reduce idle time to 90 seconds * Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server * Use a shared pool size, 512 by default, to stay below open file limit * Add some tests * Add more tests * Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds * Use a shared pool that returns preferred connection but re-purposes other ones when needed * Fix wrong connection being returned on subsequent calls within the same thread * Reduce mutex calls on flushes from 2 to 1 and add test for reaping --- app/workers/activitypub/delivery_worker.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'app/workers') diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 5e4c391f0..79f1e8153 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -17,6 +17,7 @@ class ActivityPub::DeliveryWorker @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url + @host = Addressable::URI.parse(inbox_url).normalized_site perform_request @@ -28,16 +29,18 @@ class ActivityPub::DeliveryWorker private - def build_request - request = Request.new(:post, @inbox_url, body: @json) + def build_request(http_client) + request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) request.add_headers(HEADERS) end def perform_request light = Stoplight(@inbox_url) do - build_request.perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + request_pool.with(@host) do |http_client| + build_request(http_client).perform do |response| + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + end end end @@ -57,4 +60,8 @@ class ActivityPub::DeliveryWorker def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end + + def request_pool + RequestPool.current + end end -- cgit From bc60d794f8411daa3c4c1541fb373309e4c029b6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:59:53 +0200 Subject: Change ActivityPub::DeliveryWorker to not retry HTTP 501 errors (#11233) --- app/workers/activitypub/delivery_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 79f1e8153..818fd8f5d 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -54,7 +54,7 @@ class ActivityPub::DeliveryWorker end def response_error_unsalvageable?(response) - (400...500).cover?(response.code) && ![401, 408, 429].include?(response.code) + response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) end def failure_tracker -- cgit