Slightly reduce RAM usage (#7301)

* No need to re-require sidekiq plugins, they are required via Gemfile

* Add derailed_benchmarks tool, no need to require TTY gems in Gemfile

* Replace ruby-oembed with FetchOEmbedService

Reduce startup by 45382 allocated objects

* Remove preloaded JSON-LD in favour of caching HTTP responses

Reduce boot RAM by about 6 MiB

* Fix tests

* Fix test suite by stubbing out JSON-LD contexts

17 files changed, 112 insertions, 90 deletions

diff --git a/app/controllers/api/web/embeds_controller.rb b/app/controllers/api/web/embeds_controller.rb
index f2fe74b17..987290a14 100644
--- a/app/controllers/api/web/embeds_controller.rb
+++ b/app/controllers/api/web/embeds_controller.rb
@@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController
     status = StatusFinder.new(params[:url]).status
     render json: status, serializer: OEmbedSerializer, width: 400
   rescue ActiveRecord::RecordNotFound
-    oembed = OEmbed::Providers.get(params[:url])
-    render json: Oj.dump(oembed.fields)
-  rescue OEmbed::NotFound
-    render json: {}, status: :not_found
+    oembed = FetchOEmbedService.new.call(params[:url])
+
+    if oembed
+      render json: oembed
+    else
+      render json: {}, status: :not_found
+    end
   end
 end
diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb
index 213d9e96d..91b521e7f 100644
--- a/app/controllers/settings/follower_domains_controller.rb
+++ b/app/controllers/settings/follower_domains_controller.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class Settings::FollowerDomainsController < ApplicationController
   layout 'admin'
 
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index a3cfdadb8..e9056166c 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -48,7 +48,7 @@ module JsonLdHelper
   end
 
   def canonicalize(json)
-    graph = RDF::Graph.new << JSON::LD::API.toRdf(json)
+    graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context))
     graph.dump(:normalize)
   end
 
@@ -90,4 +90,19 @@ module JsonLdHelper
     request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     request
   end
+
+  def load_jsonld_context(url, _options = {}, &_block)
+    json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do
+      request = Request.new(:get, url)
+      request.add_headers('Accept' => 'application/ld+json')
+
+      request.perform do |res|
+        raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json'
+        res.body_with_limit
+      end
+    end
+
+    doc = JSON::LD::API::RemoteDocument.new(url, json)
+    block_given? ? yield(doc) : doc
+  end
 end
diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb
deleted file mode 100644
index 3bec7211b..000000000
--- a/app/lib/provider_discovery.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-class ProviderDiscovery < OEmbed::ProviderDiscovery
-  class << self
-    def get(url, **options)
-      provider = discover_provider(url, options)
-
-      options.delete(:html)
-
-      provider.get(url, options)
-    end
-
-    def discover_provider(url, **options)
-      format = options[:format]
-
-      html = if options[:html]
-               Nokogiri::HTML(options[:html])
-             else
-               Request.new(:get, url).perform do |res|
-                 raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
-                 Nokogiri::HTML(res.body_with_limit)
-               end
-             end
-
-      if format.nil? || format == :json
-        provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
-        format ||= :json if provider_endpoint
-      end
-
-      if format.nil? || format == :xml
-        provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
-        format ||= :xml if provider_endpoint
-      end
-
-      raise OEmbed::NotFound, url if provider_endpoint.nil?
-      begin
-        provider_endpoint = Addressable::URI.parse(provider_endpoint)
-        provider_endpoint.query = nil
-        provider_endpoint = provider_endpoint.to_s
-      rescue Addressable::URI::InvalidURIError
-        raise OEmbed::NotFound, url
-      end
-
-      OEmbed::Provider.new(provider_endpoint, format)
-    end
-  end
-end
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 0f77556dc..510b80c82 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index d5920a417..77d4aa538 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_oembed
-    embed = OEmbed::Providers.get(@url, html: @html)
+    embed = FetchOEmbedService.new.call(@url, html: @html)
 
-    return false unless embed.respond_to?(:type)
+    return false if embed.nil?
 
-    @card.type          = embed.type
-    @card.title         = embed.respond_to?(:title)         ? embed.title         : ''
-    @card.author_name   = embed.respond_to?(:author_name)   ? embed.author_name   : ''
-    @card.author_url    = embed.respond_to?(:author_url)    ? embed.author_url    : ''
-    @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : ''
-    @card.provider_url  = embed.respond_to?(:provider_url)  ? embed.provider_url  : ''
+    @card.type          = embed[:type]
+    @card.title         = embed[:title]         || ''
+    @card.author_name   = embed[:author_name]   || ''
+    @card.author_url    = embed[:author_url]    || ''
+    @card.provider_name = embed[:provider_name] || ''
+    @card.provider_url  = embed[:provider_url]  || ''
     @card.width         = 0
     @card.height        = 0
 
     case @card.type
     when 'link'
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'photo'
-      return false unless embed.respond_to?(:url)
+      return false if embed[:url].blank?
 
-      @card.embed_url        = embed.url
-      @card.image_remote_url = embed.url
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
+      @card.embed_url        = embed[:url]
+      @card.image_remote_url = embed[:url]
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
     when 'video'
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
-      @card.html             = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED)
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
+      @card.html             = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'rich'
       # Most providers rely on <script> tags, which is a no-no
       return false
     end
 
     @card.save_with_optional_image!
-  rescue OEmbed::NotFound
-    false
   end
 
   def attempt_opengraph
diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb
new file mode 100644
index 000000000..998228517
--- /dev/null
+++ b/app/services/fetch_oembed_service.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class FetchOEmbedService
+  attr_reader :url, :options, :format, :endpoint_url
+
+  def call(url, options = {})
+    @url     = url
+    @options = options
+
+    discover_endpoint!
+    fetch!
+  end
+
+  private
+
+  def discover_endpoint!
+    return if html.nil?
+
+    @format = @options[:format]
+    page    = Nokogiri::HTML(html)
+
+    if @format.nil? || @format == :json
+      @endpoint_url ||= page.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
+      @format       ||= :json if @endpoint_url
+    end
+
+    if @format.nil? || @format == :xml
+      @endpoint_url ||= page.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
+      @format       ||= :xml if @endpoint_url
+    end
+
+    return if @endpoint_url.blank?
+
+    @endpoint_url = Addressable::URI.parse(@endpoint_url).to_s
+  rescue Addressable::URI::InvalidURIError
+    @endpoint_url = nil
+  end
+
+  def fetch!
+    return if @endpoint_url.blank?
+
+    body = Request.new(:get, @endpoint_url).perform do |res|
+      res.code != 200 ? nil : res.body_with_limit
+    end
+
+    validate(parse_for_format(body)) unless body.nil?
+  rescue Oj::ParseError, Ox::ParseError
+    nil
+  end
+
+  def parse_for_format(body)
+    case @format
+    when :json
+      Oj.load(body, mode: :strict)&.with_indifferent_access
+    when :xml
+      Ox.load(body, mode: :hash_no_attrs)&.with_indifferent_access&.dig(:oembed)
+    end
+  end
+
+  def validate(oembed)
+    oembed if oembed[:version] == '1.0' && oembed[:type].present?
+  end
+
+  def html
+    return @html if defined?(@html)
+
+    @html = @options[:html] || Request.new(:get, @url).perform do |res|
+      res.code != 200 || res.mime_type != 'text/html' ? nil : res.body_with_limit
+    end
+  end
+end
diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb
index 7a9d4f894..5ab16c057 100644
--- a/app/workers/scheduler/backup_cleanup_scheduler.rb
+++ b/app/workers/scheduler/backup_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::BackupCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
index 6488798cd..bab4ae886 100644
--- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
+++ b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::DoorkeeperCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/email_scheduler.rb b/app/workers/scheduler/email_scheduler.rb
index 24d0c0ebe..36866061b 100644
--- a/app/workers/scheduler/email_scheduler.rb
+++ b/app/workers/scheduler/email_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::EmailScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb
index 23fa7672b..42cf14128 100644
--- a/app/workers/scheduler/feed_cleanup_scheduler.rb
+++ b/app/workers/scheduler/feed_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::FeedCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb
index a33ca031e..613a5e336 100644
--- a/app/workers/scheduler/ip_cleanup_scheduler.rb
+++ b/app/workers/scheduler/ip_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::IpCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/media_cleanup_scheduler.rb b/app/workers/scheduler/media_cleanup_scheduler.rb
index ce32ce314..c35686fcb 100644
--- a/app/workers/scheduler/media_cleanup_scheduler.rb
+++ b/app/workers/scheduler/media_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::MediaCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
index 3b9211e81..af2ae3120 100644
--- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-scheduler'
-
 class Scheduler::SubscriptionsCleanupScheduler
   include Sidekiq::Worker
 
diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb
index 469a3d2a6..dc16e85c2 100644
--- a/app/workers/scheduler/subscriptions_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_scheduler.rb
@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-scheduler'
-require 'sidekiq-bulk'
-
 class Scheduler::SubscriptionsScheduler
   include Sidekiq::Worker
 
diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb
index a8f8fbd83..245536cea 100644
--- a/app/workers/scheduler/user_cleanup_scheduler.rb
+++ b/app/workers/scheduler/user_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::UserCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/soft_block_domain_followers_worker.rb b/app/workers/soft_block_domain_followers_worker.rb
index ce76683c5..85445c7fb 100644
--- a/app/workers/soft_block_domain_followers_worker.rb
+++ b/app/workers/soft_block_domain_followers_worker.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class SoftBlockDomainFollowersWorker
   include Sidekiq::Worker