From 503319ec8f9c387ba758a5f8ac8d0249c9420c03 Mon Sep 17 00:00:00 2001
From: Fire Demon <firedemon@creature.cafe>
Date: Thu, 20 Aug 2020 06:21:24 -0500
Subject: [Federation] Fall back to server representative if fetching a public
 resource with another local account fails

---
 app/helpers/jsonld_helper.rb           | 27 ++++++++++++++++++++++++++-
 app/services/fetch_resource_service.rb | 14 ++++++++------
 app/workers/fetch_reply_worker.rb      | 13 +++++++------
 3 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 1c473efa3..b93284637 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -76,9 +76,31 @@ module JsonLdHelper
     json.present? && json['id'] == uri ? json : nil
   end
 
+  def uri_allowed?(uri)
+    host = Addressable::URI.parse(uri)&.normalized_host
+    Rails.cache.fetch("fetch_resource:#{host}", expires_in: 1.hour) { DomainAllow.allowed?(host) }
+  rescue Addressable::URI::InvalidURIError
+    false
+  end
+
   def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
+    return unless uri_allowed?(uri)
+
     on_behalf_of ||= Account.representative
+    skip_retry = on_behalf_of.id == -99 || Rails.env.development?
 
+    begin
+      fetch_body(uri, on_behalf_of, !skip_retry || raise_on_temporary_error)
+    rescue Mastodon::UnexpectedResponseError
+      raise if skip_retry
+
+      fetch_body(uri, Account.representative, raise_on_temporary_error)
+    end
+  rescue Addressable::URI::InvalidURIError
+    nil
+  end
+
+  def fetch_body(uri, on_behalf_of, raise_on_temporary_error = false)
     build_request(uri, on_behalf_of).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
@@ -87,6 +109,9 @@ module JsonLdHelper
   end
 
   def body_to_json(body, compare_id: nil)
+    body.strip! if body.is_a?(String)
+    return if body.blank?
+
     json = body.is_a?(String) ? Oj.load(body, mode: :strict) : body
 
     return if compare_id.present? && json['id'] != compare_id
@@ -114,7 +139,7 @@ module JsonLdHelper
 
   def build_request(uri, on_behalf_of = nil)
     Request.new(:get, uri).tap do |request|
-      request.on_behalf_of(on_behalf_of) if on_behalf_of
+      request.on_behalf_of(on_behalf_of) unless Rails.env.development? || on_behalf_of.blank?
       request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     end
   end
diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb
index 3264d14c2..17e8024de 100644
--- a/app/services/fetch_resource_service.rb
+++ b/app/services/fetch_resource_service.rb
@@ -10,7 +10,7 @@ class FetchResourceService < BaseService
   def call(url, on_behalf_of: nil)
     return if url.blank?
 
-    @on_behalf_of = on_behalf_of
+    @on_behalf_of = on_behalf_of || Account.representative
 
     process(url)
   rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
@@ -20,8 +20,9 @@ class FetchResourceService < BaseService
 
   private
 
-  def process(url, terminal = false)
+  def process(url, terminal = false, retry_as_server = false)
     @url = url
+    @retry_as_server ||= retry_as_server
 
     perform_request { |response| process_response(response, terminal) }
   end
@@ -37,13 +38,14 @@ class FetchResourceService < BaseService
       # and prevents even public resources from being fetched, so
       # don't do it
 
-      request.on_behalf_of(@on_behalf_of || Account.representative) unless Rails.env.development?
+      request.on_behalf_of(@retry_as_server ? Account.representative : @on_behalf_of) unless Rails.env.development?
     end.perform(&block)
   end
 
   def process_response(response, terminal = false)
     @response_code = response.code
-    return nil if response.code != 200
+    skip_retry = @retry_as_server || Rails.env.development? || @on_behalf_of.id == -99
+    return (skip_retry ? nil : process(response.uri, terminal, true)) if response.code != 200
 
     if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
       body = response.body_with_limit
@@ -69,13 +71,13 @@ class FetchResourceService < BaseService
     page      = Nokogiri::HTML(response.body_with_limit)
     json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
 
-    process(json_link['href'], terminal: true) unless json_link.nil?
+    process(json_link['href'], true) unless json_link.nil?
   end
 
   def process_link_headers(link_header)
     json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])
 
-    process(json_link.href, terminal: true) unless json_link.nil?
+    process(json_link.href, true) unless json_link.nil?
   end
 
   def parse_link_header(response)
diff --git a/app/workers/fetch_reply_worker.rb b/app/workers/fetch_reply_worker.rb
index b93ac6c7a..67db042fd 100644
--- a/app/workers/fetch_reply_worker.rb
+++ b/app/workers/fetch_reply_worker.rb
@@ -6,11 +6,12 @@ class FetchReplyWorker
 
   sidekiq_options queue: 'pull', retry: 3
 
-  def perform(child_url)
-    if child_url.is_a?(String)
-      FetchRemoteStatusService.new.call(child_url, nil)
-    elsif child_url.is_a?(Enumerable)
-      child_url.each { |url| FetchRemoteStatusService.new.call(url, nil) }
-    end
+  def perform(child_url, account_id = nil)
+    account = account_id.blank? ? nil : Account.find_by(id: account_id)
+    on_behalf_of = account.blank? ? nil : account.followers.local.random.first
+
+    FetchRemoteStatusService.new.call(child_url, nil, on_behalf_of)
+  rescue ActiveRecord::RecordNotFound
+    nil
   end
 end
-- 
cgit