diff options
| -rw-r--r-- | app/helpers/jsonld_helper.rb | 27 | ||||
| -rw-r--r-- | app/services/fetch_resource_service.rb | 14 | ||||
| -rw-r--r-- | app/workers/fetch_reply_worker.rb | 13 |
3 files changed, 41 insertions, 13 deletions
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 1c473efa3..b93284637 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -76,9 +76,31 @@ module JsonLdHelper json.present? && json['id'] == uri ? json : nil end + def uri_allowed?(uri) + host = Addressable::URI.parse(uri)&.normalized_host + Rails.cache.fetch("fetch_resource:#{host}", expires_in: 1.hour) { DomainAllow.allowed?(host) } + rescue Addressable::URI::InvalidURIError + false + end + def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false) + return unless uri_allowed?(uri) + on_behalf_of ||= Account.representative + skip_retry = on_behalf_of.id == -99 || Rails.env.development? + begin + fetch_body(uri, on_behalf_of, !skip_retry || raise_on_temporary_error) + rescue Mastodon::UnexpectedResponseError + raise if skip_retry + + fetch_body(uri, Account.representative, raise_on_temporary_error) + end + rescue Addressable::URI::InvalidURIError + nil + end + + def fetch_body(uri, on_behalf_of, raise_on_temporary_error = false) build_request(uri, on_behalf_of).perform do |response| raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error @@ -87,6 +109,9 @@ module JsonLdHelper end def body_to_json(body, compare_id: nil) + body.strip! if body.is_a?(String) + return if body.blank? + json = body.is_a?(String) ? Oj.load(body, mode: :strict) : body return if compare_id.present? && json['id'] != compare_id @@ -114,7 +139,7 @@ module JsonLdHelper def build_request(uri, on_behalf_of = nil) Request.new(:get, uri).tap do |request| - request.on_behalf_of(on_behalf_of) if on_behalf_of + request.on_behalf_of(on_behalf_of) unless Rails.env.development? || on_behalf_of.blank? request.add_headers('Accept' => 'application/activity+json, application/ld+json') end end diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb index 3264d14c2..17e8024de 100644 --- a/app/services/fetch_resource_service.rb +++ b/app/services/fetch_resource_service.rb @@ -10,7 +10,7 @@ class FetchResourceService < BaseService def call(url, on_behalf_of: nil) return if url.blank? - @on_behalf_of = on_behalf_of + @on_behalf_of = on_behalf_of || Account.representative process(url) rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e @@ -20,8 +20,9 @@ class FetchResourceService < BaseService private - def process(url, terminal = false) + def process(url, terminal = false, retry_as_server = false) @url = url + @retry_as_server ||= retry_as_server perform_request { |response| process_response(response, terminal) } end @@ -37,13 +38,14 @@ class FetchResourceService < BaseService # and prevents even public resources from being fetched, so # don't do it - request.on_behalf_of(@on_behalf_of || Account.representative) unless Rails.env.development? + request.on_behalf_of(@retry_as_server ? Account.representative : @on_behalf_of) unless Rails.env.development? end.perform(&block) end def process_response(response, terminal = false) @response_code = response.code - return nil if response.code != 200 + skip_retry = @retry_as_server || Rails.env.development? || @on_behalf_of.id == -99 + return (skip_retry ? nil : process(response.uri, terminal, true)) if response.code != 200 if ['application/activity+json', 'application/ld+json'].include?(response.mime_type) body = response.body_with_limit @@ -69,13 +71,13 @@ class FetchResourceService < BaseService page = Nokogiri::HTML(response.body_with_limit) json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } - process(json_link['href'], terminal: true) unless json_link.nil? + process(json_link['href'], true) unless json_link.nil? end def process_link_headers(link_header) json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) - process(json_link.href, terminal: true) unless json_link.nil? + process(json_link.href, true) unless json_link.nil? end def parse_link_header(response) diff --git a/app/workers/fetch_reply_worker.rb b/app/workers/fetch_reply_worker.rb index b93ac6c7a..67db042fd 100644 --- a/app/workers/fetch_reply_worker.rb +++ b/app/workers/fetch_reply_worker.rb @@ -6,11 +6,12 @@ class FetchReplyWorker sidekiq_options queue: 'pull', retry: 3 - def perform(child_url) - if child_url.is_a?(String) - FetchRemoteStatusService.new.call(child_url, nil) - elsif child_url.is_a?(Enumerable) - child_url.each { |url| FetchRemoteStatusService.new.call(url, nil) } - end + def perform(child_url, account_id = nil) + account = account_id.blank? ? nil : Account.find_by(id: account_id) + on_behalf_of = account.blank? ? nil : account.followers.local.random.first + + FetchRemoteStatusService.new.call(child_url, nil, on_behalf_of) + rescue ActiveRecord::RecordNotFound + nil end end |