Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- README.md
  discarded upstream changes
- app/controllers/api/v1/bookmarks_controller.rb
  finally merged upstream, some code style fixes
  and slightly changed pagination code
- app/controllers/application_controller.rb
  changed upstream to always return HTML error pages
  slight conflict caused by theming code
- app/models/bookmark.rb
  finally merged upstream, no real conflict
- spec/controllers/api/v1/bookmarks_controller_spec.rb
  finally merged upstream, slightly changed pagination code

3 files changed, 52 insertions, 8 deletions

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 640c60fd4..29880e8d8 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -39,6 +39,12 @@ class FetchLinkCardService < BaseService
   def process_url
     @card ||= PreviewCard.new(url: @url)
 
+    attempt_oembed || attempt_opengraph
+  end
+
+  def html
+    return @html if defined?(@html)
+
     Request.new(:get, @url).perform do |res|
       if res.code == 200 && res.mime_type == 'text/html'
         @html = res.body_with_limit
@@ -48,10 +54,6 @@ class FetchLinkCardService < BaseService
         @html_charset = nil
       end
     end
-
-    return if @html.nil?
-
-    attempt_oembed || attempt_opengraph
   end
 
   def attach_card
@@ -88,12 +90,17 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_oembed
-    service = FetchOEmbedService.new
-    embed   = service.call(@url, html: @html)
-    url     = Addressable::URI.parse(service.endpoint_url)
+    service         = FetchOEmbedService.new
+    url_domain      = Addressable::URI.parse(@url).normalized_host
+    cached_endpoint = Rails.cache.read("oembed_endpoint:#{url_domain}")
+
+    embed   = service.call(@url, cached_endpoint: cached_endpoint) unless cached_endpoint.nil?
+    embed ||= service.call(@url, html: html) unless html.nil?
 
     return false if embed.nil?
 
+    url = Addressable::URI.parse(service.endpoint_url)
+
     @card.type          = embed[:type]
     @card.title         = embed[:title]         || ''
     @card.author_name   = embed[:author_name]   || ''
@@ -127,6 +134,8 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_opengraph
+    return if html.nil?
+
     detector = CharlockHolmes::EncodingDetector.new
     detector.strip_tags = true
 
diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb
index 10176cfb9..4f8498c62 100644
--- a/app/services/fetch_oembed_service.rb
+++ b/app/services/fetch_oembed_service.rb
@@ -1,13 +1,20 @@
 # frozen_string_literal: true
 
 class FetchOEmbedService
+  ENDPOINT_CACHE_EXPIRES_IN = 24.hours.freeze
+
   attr_reader :url, :options, :format, :endpoint_url
 
   def call(url, options = {})
     @url     = url
     @options = options
 
-    discover_endpoint!
+    if @options[:cached_endpoint]
+      parse_cached_endpoint!
+    else
+      discover_endpoint!
+    end
+
     fetch!
   end
 
@@ -32,10 +39,32 @@ class FetchOEmbedService
     return if @endpoint_url.blank?
 
     @endpoint_url = (Addressable::URI.parse(@url) + @endpoint_url).to_s
+
+    cache_endpoint!
   rescue Addressable::URI::InvalidURIError
     @endpoint_url = nil
   end
 
+  def parse_cached_endpoint!
+    cached = @options[:cached_endpoint]
+
+    return if cached[:endpoint].nil? || cached[:format].nil?
+
+    @endpoint_url = Addressable::Template.new(cached[:endpoint]).expand(url: @url).to_s
+    @format       = cached[:format]
+  end
+
+  def cache_endpoint!
+    url_domain = Addressable::URI.parse(@url).normalized_host
+
+    endpoint_hash = {
+      endpoint: @endpoint_url.gsub(URI.encode_www_form_component(@url), '{url}'),
+      format: @format,
+    }
+
+    Rails.cache.write("oembed_endpoint:#{url_domain}", endpoint_hash, expires_in: ENDPOINT_CACHE_EXPIRES_IN)
+  end
+
   def fetch!
     return if @endpoint_url.blank?
 
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index aa883597a..4e971a4b8 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -24,6 +24,12 @@ class ResolveURLService < BaseService
       status = FetchRemoteStatusService.new.call(resource_url, body, protocol)
       authorize_with @on_behalf_of, status, :show? unless status.nil?
       status
+    elsif fetched_resource.nil? && @on_behalf_of.present?
+      # It may happen that the resource is a private toot, and thus not fetchable,
+      # but we can return the toot if we already know about it.
+      status = Status.find_by(uri: @url) || Status.find_by(url: @url)
+      authorize_with @on_behalf_of, status, :show? unless status.nil?
+      status
     end
   end