Fix #2572 - Resolve preview cards for remote statuses as well as local ones (#3088)

2 files changed, 28 insertions, 2 deletions

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 27d7c2a70..dec4aabd6 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -7,11 +7,11 @@ class FetchLinkCardService < BaseService
 
   def call(status)
     # Get first http/https URL that isn't local
-    url = status.text.match(URL_PATTERN).to_a.reject { |uri| TagManager.instance.local_url?(uri) }.first
+    url = parse_urls(status)
 
     return if url.nil?
 
-    url  = Addressable::URI.parse(url).normalize.to_s
+    url  = url.to_s
     card = PreviewCard.where(status: status).first_or_initialize(status: status, url: url)
     res  = http_client.head(url)
 
@@ -22,6 +22,28 @@ class FetchLinkCardService < BaseService
 
   private
 
+  def parse_urls(status)
+    if status.local?
+      urls = status.text.match(URL_PATTERN).to_a.map { |uri| Addressable::URI.parse(uri).normalize }
+    else
+      html  = Nokogiri::HTML(status.text)
+      links = html.css('a')
+      urls  = links.map { |a| Addressable::URI.parse(a['href']).normalize unless skip_link?(a) }.compact
+    end
+
+    urls.reject { |uri| bad_url?(uri) }.first
+  end
+
+  def bad_url?(uri)
+    # Avoid local instance URLs and invalid URLs
+    TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.host.blank?
+  end
+
+  def skip_link?(a)
+    # Avoid links for hashtags and mentions (microformats)
+    a['rel']&.include?('tag') || a['class']&.include?('u-url')
+  end
+
   def attempt_oembed(card, url)
     response = OEmbed::Providers.get(url)
 
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
index c573c0490..04d6a100f 100644
--- a/app/services/process_feed_service.rb
+++ b/app/services/process_feed_service.rb
@@ -69,8 +69,12 @@ class ProcessFeedService < BaseService
 
       notify_about_mentions!(status) unless status.reblog?
       notify_about_reblog!(status) if status.reblog? && status.reblog.account.local?
+
       Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
+
+      LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text.present?
       DistributionWorker.perform_async(status.id)
+
       status
     end