OEmbed support for PreviewCard (#2337)

* OEmbed support for PreviewCard

* Improve ProviderDiscovery code failure treatment

* Do not crawl links if there is a content warning, since those
don't display a link card anyway

* Reset db schema

* Fresh migrate

* Fix rubocop style issues
Fix #1681 - return existing access token when applicable instead of creating new

* Fix test

* Extract http client to helper

* Improve oembed controller

4 files changed, 48 insertions, 18 deletions

diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb
index c3dad1eb9..9c514aa9f 100644
--- a/app/services/fetch_atom_service.rb
+++ b/app/services/fetch_atom_service.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class FetchAtomService < BaseService
+  include HttpHelper
+
   def call(url)
     return if url.blank?
 
@@ -45,8 +47,4 @@ class FetchAtomService < BaseService
   def fetch(url)
     http_client.get(url).to_s
   end
-
-  def http_client
-    HTTP.timeout(:per_operation, write: 10, connect: 10, read: 10).follow
-  end
 end
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index f74a0d34d..416c5fdad 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
 class FetchLinkCardService < BaseService
+  include HttpHelper
+
   URL_PATTERN = %r{https?://\S+}
-  USER_AGENT = "#{HTTP::Request::USER_AGENT} (Mastodon/#{Mastodon::VERSION}; +http://#{Rails.configuration.x.local_domain}/)"
 
   def call(status)
     # Get first http/https URL that isn't local
@@ -10,13 +11,53 @@ class FetchLinkCardService < BaseService
 
     return if url.nil?
 
+    card = PreviewCard.where(status: status).first_or_initialize(status: status, url: url)
+    attempt_opengraph(card, url) unless attempt_oembed(card, url)
+  end
+
+  private
+
+  def attempt_oembed(card, url)
+    response = OEmbed::Providers.get(url)
+
+    card.type          = response.type
+    card.title         = response.respond_to?(:title)         ? response.title         : ''
+    card.author_name   = response.respond_to?(:author_name)   ? response.author_name   : ''
+    card.author_url    = response.respond_to?(:author_url)    ? response.author_url    : ''
+    card.provider_name = response.respond_to?(:provider_name) ? response.provider_name : ''
+    card.provider_url  = response.respond_to?(:provider_url)  ? response.provider_url  : ''
+    card.width         = 0
+    card.height        = 0
+
+    case card.type
+    when 'link'
+      card.image = URI.parse(response.thumbnail_url) if response.respond_to?(:thumbnail_url)
+    when 'photo'
+      card.url    = response.url
+      card.width  = response.width.presence  || 0
+      card.height = response.height.presence || 0
+    when 'video'
+      card.width  = response.width.presence  || 0
+      card.height = response.height.presence || 0
+      card.html   = Formatter.instance.sanitize(response.html, Sanitize::Config::MASTODON_OEMBED)
+    when 'rich'
+      # Most providers rely on <script> tags, which is a no-no
+      return false
+    end
+
+    card.save_with_optional_image!
+  rescue OEmbed::NotFound
+    false
+  end
+
+  def attempt_opengraph(card, url)
     response = http_client.get(url)
 
     return if response.code != 200 || response.mime_type != 'text/html'
 
     page = Nokogiri::HTML(response.to_s)
-    card = PreviewCard.where(status: status).first_or_initialize(status: status, url: url)
 
+    card.type        = :link
     card.title       = meta_property(page, 'og:title') || page.at_xpath('//title')&.content
     card.description = meta_property(page, 'og:description') || meta_property(page, 'description')
     card.image       = URI.parse(Addressable::URI.parse(meta_property(page, 'og:image')).normalize.to_s) if meta_property(page, 'og:image')
@@ -26,12 +67,6 @@ class FetchLinkCardService < BaseService
     card.save_with_optional_image!
   end
 
-  private
-
-  def http_client
-    HTTP.headers(user_agent: USER_AGENT).timeout(:per_operation, write: 10, connect: 10, read: 10).follow
-  end
-
   def meta_property(html, property)
     html.at_xpath("//meta[@property=\"#{property}\"]")&.attribute('content')&.value || html.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
   end
diff --git a/app/services/follow_remote_account_service.rb b/app/services/follow_remote_account_service.rb
index fbf983093..0b5abf58a 100644
--- a/app/services/follow_remote_account_service.rb
+++ b/app/services/follow_remote_account_service.rb
@@ -2,6 +2,7 @@
 
 class FollowRemoteAccountService < BaseService
   include OStatus2::MagicKey
+  include HttpHelper
 
   DFRN_NS = 'http://purl.org/macgirvin/dfrn/1.0'
 
@@ -73,7 +74,7 @@ class FollowRemoteAccountService < BaseService
   end
 
   def get_feed(url)
-    response = http_client.get(Addressable::URI.parse(url).normalize)
+    response = http_client(write: 20, connect: 20, read: 50).get(Addressable::URI.parse(url).normalize)
     [response.to_s, Nokogiri::XML(response)]
   end
 
@@ -98,8 +99,4 @@ class FollowRemoteAccountService < BaseService
   def get_profile(body, account)
     RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), false)
   end
-
-  def http_client
-    HTTP.timeout(:per_operation, write: 20, connect: 20, read: 50)
-  end
 end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 30b8032ed..b665391e3 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -34,7 +34,7 @@ class PostStatusService < BaseService
     process_mentions_service.call(status)
     process_hashtags_service.call(status)
 
-    LinkCrawlWorker.perform_async(status.id)
+    LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text.present?
     DistributionWorker.perform_async(status.id)
     Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)