Merge remote-tracking branch 'glitch/main' HEAD main

43 files changed, 416 insertions, 492 deletions

diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb
index 6d3401b37..8829d8605 100644
--- a/app/lib/activity_tracker.rb
+++ b/app/lib/activity_tracker.rb
@@ -27,14 +27,12 @@ class ActivityTracker
     (start_at.to_date...end_at.to_date).map do |date|
       key = key_at(date.to_time(:utc))
 
-      value = begin
-        case @type
-        when :basic
-          redis.get(key).to_i
-        when :unique
-          redis.pfcount(key)
-        end
-      end
+      value = case @type
+              when :basic
+                redis.get(key).to_i
+              when :unique
+                redis.pfcount(key)
+              end
 
       [date, value]
     end
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index f4c67cccd..5d9596254 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -106,7 +106,8 @@ class ActivityPub::Activity
       actor_id = value_or_id(first_of_value(@object['attributedTo']))
 
       if actor_id == @account.uri
-        return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform
+        virtual_object = { 'type' => 'Create', 'actor' => actor_id, 'object' => @object }
+        return ActivityPub::Activity.factory(virtual_object, @account, request_id: @options[:request_id]).perform
       end
     end
 
@@ -152,9 +153,10 @@ class ActivityPub::Activity
   def fetch_remote_original_status
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first)
+
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
     elsif @object['url'].present?
-      ::FetchRemoteStatusService.new.call(@object['url'])
+      ::FetchRemoteStatusService.new.call(@object['url'], request_id: @options[:request_id])
     end
   end
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 4dfbfc665..eca446243 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -108,26 +108,24 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def process_status_params
     @status_parser = ActivityPub::Parser::StatusParser.new(@json, followers_collection: @account.followers_url)
 
-    @params = begin
-      {
-        uri: @status_parser.uri,
-        url: @status_parser.url || @status_parser.uri,
-        account: @account,
-        text: converted_object_type? ? converted_text : (@status_parser.text || ''),
-        language: @status_parser.language,
-        spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''),
-        created_at: @status_parser.created_at,
-        edited_at: @status_parser.edited_at && @status_parser.edited_at != @status_parser.created_at ? @status_parser.edited_at : nil,
-        override_timestamps: @options[:override_timestamps],
-        reply: @status_parser.reply,
-        sensitive: @account.sensitized? || @status_parser.sensitive || false,
-        visibility: @status_parser.visibility,
-        thread: replied_to_status,
-        conversation: conversation_from_uri(@object['conversation']),
-        media_attachment_ids: process_attachments.take(4).map(&:id),
-        poll: process_poll,
-      }
-    end
+    @params = {
+      uri: @status_parser.uri,
+      url: @status_parser.url || @status_parser.uri,
+      account: @account,
+      text: converted_object_type? ? converted_text : (@status_parser.text || ''),
+      language: @status_parser.language,
+      spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''),
+      created_at: @status_parser.created_at,
+      edited_at: @status_parser.edited_at && @status_parser.edited_at != @status_parser.created_at ? @status_parser.edited_at : nil,
+      override_timestamps: @options[:override_timestamps],
+      reply: @status_parser.reply,
+      sensitive: @account.sensitized? || @status_parser.sensitive || false,
+      visibility: @status_parser.visibility,
+      thread: replied_to_status,
+      conversation: conversation_from_uri(@object['conversation']),
+      media_attachment_ids: process_attachments.take(4).map(&:id),
+      poll: process_poll,
+    }
   end
 
   def process_audience
@@ -285,7 +283,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     media_attachments
   rescue Addressable::URI::InvalidURIError => e
-    Rails.logger.debug "Invalid URL in attachment: #{e}"
+    Rails.logger.debug { "Invalid URL in attachment: #{e}" }
     media_attachments
   end
 
@@ -327,18 +325,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def resolve_thread(status)
     return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
 
-    ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
+    ThreadResolveWorker.perform_async(status.id, in_reply_to_uri, { 'request_id' => @options[:request_id] })
   end
 
   def fetch_replies(status)
     collection = @object['replies']
     return if collection.nil?
 
-    replies = ActivityPub::FetchRepliesService.new.call(status, collection, false)
+    replies = ActivityPub::FetchRepliesService.new.call(status, collection, allow_synchronous_requests: false, request_id: @options[:request_id])
     return unless replies.nil?
 
     uri = value_or_id(collection)
-    ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
+    ActivityPub::FetchRepliesWorker.perform_async(status.id, uri, { 'request_id' => @options[:request_id] }) unless uri.nil?
   end
 
   def conversation_from_uri(uri)
diff --git a/app/lib/activitypub/case_transform.rb b/app/lib/activitypub/case_transform.rb
index 7f716f862..d36e01b8f 100644
--- a/app/lib/activitypub/case_transform.rb
+++ b/app/lib/activitypub/case_transform.rb
@@ -13,7 +13,7 @@ module ActivityPub::CaseTransform
       when Symbol then camel_lower(value.to_s).to_sym
       when String
         camel_lower_cache[value] ||= if value.start_with?('_:')
-                                       '_:' + value.gsub(/\A_:/, '').underscore.camelize(:lower)
+                                       "_:#{value.gsub(/\A_:/, '').underscore.camelize(:lower)}"
                                      else
                                        value.underscore.camelize(:lower)
                                      end
diff --git a/app/lib/activitypub/forwarder.rb b/app/lib/activitypub/forwarder.rb
index 4206b9d82..3a94f9669 100644
--- a/app/lib/activitypub/forwarder.rb
+++ b/app/lib/activitypub/forwarder.rb
@@ -12,7 +12,7 @@ class ActivityPub::Forwarder
   end
 
   def forward!
-    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
+    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
       [payload, signature_account_id, inbox_url]
     end
   end
@@ -28,13 +28,11 @@ class ActivityPub::Forwarder
   end
 
   def signature_account_id
-    @signature_account_id ||= begin
-      if in_reply_to_local?
-        in_reply_to.account_id
-      else
-        reblogged_by_account_ids.first
-      end
-    end
+    @signature_account_id ||= if in_reply_to_local?
+                                in_reply_to.account_id
+                              else
+                                reblogged_by_account_ids.first
+                              end
   end
 
   def inboxes
diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb
index f90adaf6c..ea59879f3 100644
--- a/app/lib/activitypub/linked_data_signature.rb
+++ b/app/lib/activitypub/linked_data_signature.rb
@@ -27,14 +27,12 @@ class ActivityPub::LinkedDataSignature
     document_hash  = hash(@json.without('signature'))
     to_be_verified = options_hash + document_hash
 
-    if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
-      creator
-    end
+    creator if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
   end
 
   def sign!(creator, sign_with: nil)
     options = {
-      'type'    => 'RsaSignature2017',
+      'type' => 'RsaSignature2017',
       'creator' => ActivityPub::TagManager.instance.key_uri_for(creator),
       'created' => Time.now.utc.iso8601,
     }
diff --git a/app/lib/activitypub/parser/media_attachment_parser.rb b/app/lib/activitypub/parser/media_attachment_parser.rb
index 656be84b7..56b8b23f8 100644
--- a/app/lib/activitypub/parser/media_attachment_parser.rb
+++ b/app/lib/activitypub/parser/media_attachment_parser.rb
@@ -50,9 +50,7 @@ class ActivityPub::Parser::MediaAttachmentParser
     components = begin
       blurhash = @json['blurhash']
 
-      if blurhash.present? && /^[\w#$%*+,-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
-        Blurhash.components(blurhash)
-      end
+      Blurhash.components(blurhash) if blurhash.present? && /^[\w#$%*+,-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
     end
 
     components.present? && components.none? { |comp| comp > 5 }
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 3d6b28ef5..a65a9565a 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -26,6 +26,7 @@ class ActivityPub::TagManager
       target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
+
       short_account_status_url(target.account, target)
     end
   end
@@ -38,6 +39,7 @@ class ActivityPub::TagManager
       target.instance_actor? ? instance_actor_url : account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
+
       account_status_url(target.account, target)
     when :emoji
       emoji_url(target)
diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb
index 816615f99..9ab3776c9 100644
--- a/app/lib/admin/metrics/dimension/software_versions_dimension.rb
+++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb
@@ -58,12 +58,10 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim
   end
 
   def redis_info
-    @redis_info ||= begin
-      if redis.is_a?(Redis::Namespace)
-        redis.redis.info
-      else
-        redis.info
-      end
-    end
+    @redis_info ||= if redis.is_a?(Redis::Namespace)
+                      redis.redis.info
+                    else
+                      redis.info
+                    end
   end
 end
diff --git a/app/lib/admin/metrics/dimension/space_usage_dimension.rb b/app/lib/admin/metrics/dimension/space_usage_dimension.rb
index 5867c5bab..cc8560890 100644
--- a/app/lib/admin/metrics/dimension/space_usage_dimension.rb
+++ b/app/lib/admin/metrics/dimension/space_usage_dimension.rb
@@ -59,12 +59,10 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension
   end
 
   def redis_info
-    @redis_info ||= begin
-      if redis.is_a?(Redis::Namespace)
-        redis.redis.info
-      else
-        redis.info
-      end
-    end
+    @redis_info ||= if redis.is_a?(Redis::Namespace)
+                      redis.redis.info
+                    else
+                      redis.info
+                    end
   end
 end
diff --git a/app/lib/admin/metrics/retention.rb b/app/lib/admin/metrics/retention.rb
index f6135ac1e..9bd47c58e 100644
--- a/app/lib/admin/metrics/retention.rb
+++ b/app/lib/admin/metrics/retention.rb
@@ -42,25 +42,54 @@ class Admin::Metrics::Retention
   end
 
   def perform_query
-    sql = <<-SQL.squish
+    report_rows.each_with_object([]) do |row, arr|
+      current_cohort = arr.last
+
+      if current_cohort.nil? || current_cohort.period != row['cohort_period']
+        current_cohort = Cohort.new(period: row['cohort_period'], frequency: @frequency, data: [])
+        arr << current_cohort
+      end
+
+      value, rate = row['retention_value_and_rate'].delete('{}').split(',')
+
+      current_cohort.data << CohortData.new(
+        date: row['retention_period'],
+        rate: rate.to_f,
+        value: value.to_s
+      )
+    end
+  end
+
+  def report_rows
+    ActiveRecord::Base.connection.select_all(sanitized_sql_string)
+  end
+
+  def sanitized_sql_string
+    ActiveRecord::Base.sanitize_sql_array(
+      [sql_query_string, { start_at: @start_at, end_at: @end_at, frequency: @frequency }]
+    )
+  end
+
+  def sql_query_string
+    <<~SQL.squish
       SELECT axis.*, (
         WITH new_users AS (
           SELECT users.id
           FROM users
-          WHERE date_trunc($3, users.created_at)::date = axis.cohort_period
+          WHERE date_trunc(:frequency, users.created_at)::date = axis.cohort_period
         ),
         retained_users AS (
           SELECT users.id
           FROM users
           INNER JOIN new_users on new_users.id = users.id
-          WHERE date_trunc($3, users.current_sign_in_at) >= axis.retention_period
+          WHERE date_trunc(:frequency, users.current_sign_in_at) >= axis.retention_period
         )
         SELECT ARRAY[count(*), (count(*))::float / (SELECT GREATEST(count(*), 1) FROM new_users)] AS retention_value_and_rate
         FROM retained_users
       )
       FROM (
         WITH cohort_periods AS (
-          SELECT generate_series(date_trunc($3, $1::timestamp)::date, date_trunc($3, $2::timestamp)::date, ('1 ' || $3)::interval) AS cohort_period
+          SELECT generate_series(date_trunc(:frequency, :start_at::timestamp)::date, date_trunc(:frequency, :end_at::timestamp)::date, ('1 ' || :frequency)::interval) AS cohort_period
         ),
         retention_periods AS (
           SELECT cohort_period AS retention_period FROM cohort_periods
@@ -70,24 +99,5 @@ class Admin::Metrics::Retention
         WHERE retention_period >= cohort_period
       ) as axis
     SQL
-
-    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @frequency]])
-
-    rows.each_with_object([]) do |row, arr|
-      current_cohort = arr.last
-
-      if current_cohort.nil? || current_cohort.period != row['cohort_period']
-        current_cohort = Cohort.new(period: row['cohort_period'], frequency: @frequency, data: [])
-        arr << current_cohort
-      end
-
-      value, rate = row['retention_value_and_rate'].delete('{}').split(',')
-
-      current_cohort.data << CohortData.new(
-        date: row['retention_period'],
-        rate: rate.to_f,
-        value: value.to_s
-      )
-    end
   end
 end
diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb
index f512635ab..89dfcef9f 100644
--- a/app/lib/admin/system_check.rb
+++ b/app/lib/admin/system_check.rb
@@ -2,6 +2,7 @@
 
 class Admin::SystemCheck
   ACTIVE_CHECKS = [
+    Admin::SystemCheck::MediaPrivacyCheck,
     Admin::SystemCheck::DatabaseSchemaCheck,
     Admin::SystemCheck::SidekiqProcessCheck,
     Admin::SystemCheck::RulesCheck,
diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb
index 7f922978f..0b55be350 100644
--- a/app/lib/admin/system_check/elasticsearch_check.rb
+++ b/app/lib/admin/system_check/elasticsearch_check.rb
@@ -30,19 +30,24 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
 
   def running_version
     @running_version ||= begin
-      Chewy.client.info['version']['minimum_wire_compatibility_version'] ||
-        Chewy.client.info['version']['number']
-    rescue Faraday::ConnectionFailed
+      Chewy.client.info['version']['number']
+    rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
       nil
     end
   end
 
+  def compatible_wire_version
+    Chewy.client.info['version']['minimum_wire_compatibility_version']
+  end
+
   def required_version
     '7.x'
   end
 
   def compatible_version?
     return false if running_version.nil?
-    Gem::Version.new(running_version) >= Gem::Version.new(required_version)
+
+    Gem::Version.new(running_version) >= Gem::Version.new(required_version) ||
+      Gem::Version.new(compatible_wire_version) >= Gem::Version.new(required_version)
   end
 end
diff --git a/app/lib/admin/system_check/media_privacy_check.rb b/app/lib/admin/system_check/media_privacy_check.rb
new file mode 100644
index 000000000..1df05b120
--- /dev/null
+++ b/app/lib/admin/system_check/media_privacy_check.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck
+  include RoutingHelper
+
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
+  def pass?
+    check_media_uploads!
+    @failure_message.nil?
+  end
+
+  def message
+    Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true)
+  end
+
+  private
+
+  def check_media_uploads!
+    if Rails.configuration.x.use_s3
+      check_media_listing_inaccessible_s3!
+    else
+      check_media_listing_inaccessible!
+    end
+  end
+
+  def check_media_listing_inaccessible!
+    full_url = full_asset_url(media_attachment.file.url(:original, false))
+
+    # Check if we can list the uploaded file. If true, that's an error
+    directory_url = Addressable::URI.parse(full_url)
+    directory_url.query = nil
+    filename = directory_url.path.gsub(%r{.*/}, '')
+    directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/')
+    Request.new(:get, directory_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?(filename)
+        @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error
+        @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS'
+      end
+    end
+  rescue
+    nil
+  end
+
+  def check_media_listing_inaccessible_s3!
+    urls_to_check = []
+    paperclip_options = Paperclip::Attachment.default_options
+    s3_protocol = paperclip_options[:s3_protocol]
+    s3_host_alias = paperclip_options[:s3_host_alias]
+    s3_host_name  = paperclip_options[:s3_host_name]
+    bucket_name = paperclip_options.dig(:s3_credentials, :bucket)
+
+    urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present?
+    urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/"
+    urls_to_check.uniq.each do |full_url|
+      check_s3_listing!(full_url)
+      break if @failure_message.present?
+    end
+  rescue
+    nil
+  end
+
+  def check_s3_listing!(full_url)
+    bucket_url = Addressable::URI.parse(full_url)
+    bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original))
+    bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}"
+    Request.new(:get, bucket_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?('ListBucketResult')
+        @failure_message = :upload_check_privacy_error_object_storage
+        @failure_action  = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3'
+      end
+    end
+  end
+
+  def media_attachment
+    @media_attachment ||= begin
+      attachment = Account.representative.media_attachments.first
+      if attachment.present?
+        attachment.touch # rubocop:disable Rails/SkipsModelValidations
+        attachment
+      else
+        create_test_attachment!
+      end
+    end
+  end
+
+  def create_test_attachment!
+    Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file|
+      tmp_file.write(
+        Base64.decode64(
+          '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \
+          'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \
+          'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \
+          'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \
+          'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \
+          'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q=='
+        )
+      )
+      tmp_file.flush
+      Account.representative.media_attachments.create!(file: tmp_file)
+    end
+  end
+end
diff --git a/app/lib/admin/system_check/message.rb b/app/lib/admin/system_check/message.rb
index bfcad3bf3..ad8d4b607 100644
--- a/app/lib/admin/system_check/message.rb
+++ b/app/lib/admin/system_check/message.rb
@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::Message
-  attr_reader :key, :value, :action
+  attr_reader :key, :value, :action, :critical
 
-  def initialize(key, value = nil, action = nil)
-    @key    = key
-    @value  = value
-    @action = action
+  def initialize(key, value = nil, action = nil, critical = false)
+    @key      = key
+    @value    = value
+    @action   = action
+    @critical = critical
   end
 end
diff --git a/app/lib/advanced_text_formatter.rb b/app/lib/advanced_text_formatter.rb
index 21e81d4d1..cdf1e2d9c 100644
--- a/app/lib/advanced_text_formatter.rb
+++ b/app/lib/advanced_text_formatter.rb
@@ -15,6 +15,7 @@ class AdvancedTextFormatter < TextFormatter
 
     def autolink(link, link_type)
       return link if link_type == :email
+
       @format_link.call(link)
     end
   end
diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb
index 66c1fd8c0..c90716632 100644
--- a/app/lib/delivery_failure_tracker.rb
+++ b/app/lib/delivery_failure_tracker.rb
@@ -6,7 +6,7 @@ class DeliveryFailureTracker
   FAILURE_DAYS_THRESHOLD = 7
 
   def initialize(url_or_host)
-    @host = url_or_host.start_with?('https://') || url_or_host.start_with?('http://') ? Addressable::URI.parse(url_or_host).normalized_host : url_or_host
+    @host = url_or_host.start_with?('https://', 'http://') ? Addressable::URI.parse(url_or_host).normalized_host : url_or_host
   end
 
   def track_failure!
diff --git a/app/lib/extractor.rb b/app/lib/extractor.rb
index aea60dae5..540bbe1a9 100644
--- a/app/lib/extractor.rb
+++ b/app/lib/extractor.rb
@@ -8,17 +8,15 @@ module Extractor
   module_function
 
   def extract_entities_with_indices(text, options = {}, &block)
-    entities = begin
-      extract_urls_with_indices(text, options) +
-        extract_hashtags_with_indices(text, check_url_overlap: false) +
-        extract_mentions_or_lists_with_indices(text) +
-        extract_extra_uris_with_indices(text)
-    end
+    entities = extract_urls_with_indices(text, options) +
+               extract_hashtags_with_indices(text, check_url_overlap: false) +
+               extract_mentions_or_lists_with_indices(text) +
+               extract_extra_uris_with_indices(text)
 
     return [] if entities.empty?
 
     entities = remove_overlapping_entities(entities)
-    entities.each(&block) if block_given?
+    entities.each(&block) if block
     entities
   end
 
@@ -29,7 +27,7 @@ module Extractor
 
     text.scan(Account::MENTION_RE) do |screen_name, _|
       match_data = $LAST_MATCH_INFO
-      after      = $'
+      after      = ::Regexp.last_match.post_match
 
       unless Twitter::TwitterText::Regex[:end_mention_match].match?(after)
         _, domain = screen_name.split('@')
@@ -64,7 +62,7 @@ module Extractor
       match_data     = $LAST_MATCH_INFO
       start_position = match_data.char_begin(1) - 1
       end_position   = match_data.char_end(1)
-      after          = $'
+      after          = ::Regexp.last_match.post_match
 
       if %r{\A://}.match?(after)
         hash_text.match(/(.+)(https?\Z)/) do |matched|
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 14208e557..15ff6d15f 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -7,7 +7,7 @@ class FeedManager
   include Redisable
 
   # Maximum number of items stored in a single feed
-  MAX_ITEMS = 400
+  MAX_ITEMS = 800
 
   # Number of items in the feed since last reblog of status
   # before the new reblog will be inserted. Must be <= MAX_ITEMS
@@ -306,6 +306,7 @@ class FeedManager
 
       statuses.each do |status|
         next if filter_from_direct?(status, account)
+
         added += 1 if add_to_feed(:direct, account.id, status)
       end
 
@@ -322,27 +323,27 @@ class FeedManager
   def clean_feeds!(type, ids)
     reblogged_id_sets = {}
 
-    redis.pipelined do
+    redis.pipelined do |pipeline|
       ids.each do |feed_id|
-        redis.del(key(type, feed_id))
         reblog_key = key(type, feed_id, 'reblogs')
         # We collect a future for this: we don't block while getting
         # it, but we can iterate over it later.
-        reblogged_id_sets[feed_id] = redis.zrange(reblog_key, 0, -1)
-        redis.del(reblog_key)
+        reblogged_id_sets[feed_id] = pipeline.zrange(reblog_key, 0, -1)
+        pipeline.del(key(type, feed_id), reblog_key)
       end
     end
 
     # Remove all of the reblog tracking keys we just removed the
     # references to.
-    redis.pipelined do
-      reblogged_id_sets.each do |feed_id, future|
-        future.value.each do |reblogged_id|
-          reblog_set_key = key(type, feed_id, "reblogs:#{reblogged_id}")
-          redis.del(reblog_set_key)
-        end
+    keys_to_delete = reblogged_id_sets.flat_map do |feed_id, future|
+      future.value.map do |reblogged_id|
+        key(type, feed_id, "reblogs:#{reblogged_id}")
       end
     end
+
+    redis.del(keys_to_delete) unless keys_to_delete.empty?
+
+    nil
   end
 
   private
@@ -459,6 +460,7 @@ class FeedManager
   # @return [Boolean]
   def filter_from_direct?(status, receiver_id)
     return false if receiver_id == status.account_id
+
     filter_from_mentions?(status, receiver_id)
   end
 
diff --git a/app/lib/html_aware_formatter.rb b/app/lib/html_aware_formatter.rb
index 7a1cd0340..8766c5ee0 100644
--- a/app/lib/html_aware_formatter.rb
+++ b/app/lib/html_aware_formatter.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class HtmlAwareFormatter
+  STATUS_MIME_TYPES = %w(text/plain text/markdown text/html).freeze
+
   attr_reader :text, :local, :options
 
   alias local? local
diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb
index ea522c600..0cd1d3422 100644
--- a/app/lib/importer/base_importer.rb
+++ b/app/lib/importer/base_importer.rb
@@ -45,7 +45,7 @@ class Importer::BaseImporter
   # Remove documents from the index that no longer exist in the database
   def clean_up!
     index.scroll_batches do |documents|
-      ids           = documents.map { |doc| doc['_id'] }
+      ids           = documents.pluck('_id')
       existence_map = index.adapter.target.where(id: ids).pluck(:id).each_with_object({}) { |id, map| map[id.to_s] = true }
       tmp           = ids.reject { |id| existence_map[id] }
 
diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb
index 5b5153d5c..b0721c2e0 100644
--- a/app/lib/importer/statuses_index_importer.rb
+++ b/app/lib/importer/statuses_index_importer.rb
@@ -24,13 +24,11 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
           # is called before rendering the data and we need to filter based
           # on the results of the filter, so this filtering happens here instead
           bulk.map! do |entry|
-            new_entry = begin
-              if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
-                { delete: entry[:index].except(:data) }
-              else
-                entry
-              end
-            end
+            new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
+                          { delete: entry[:index].except(:data) }
+                        else
+                          entry
+                        end
 
             if new_entry[:index]
               indexed += 1
diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index b0c4e4f42..f8a0be636 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -188,7 +188,7 @@ class LinkDetailsExtractor
   end
 
   def language
-    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
+    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').pick('lang'))
   end
 
   def icon
@@ -220,38 +220,36 @@ class LinkDetailsExtractor
   end
 
   def link_tag(name)
-    document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
+    document.xpath("//link[@rel=\"#{name}\"]").pick('href')
   end
 
   def opengraph_tag(name)
-    document.xpath("//meta[@property=\"#{name}\" or @name=\"#{name}\"]").map { |meta| meta['content'] }.first
+    document.xpath("//meta[@property=\"#{name}\" or @name=\"#{name}\"]").pick('content')
   end
 
   def meta_tag(name)
-    document.xpath("//meta[@name=\"#{name}\"]").map { |meta| meta['content'] }.first
+    document.xpath("//meta[@name=\"#{name}\"]").pick('content')
   end
 
   def structured_data
-    @structured_data ||= begin
-      # Some publications have more than one JSON-LD definition on the page,
-      # and some of those definitions aren't valid JSON either, so we have
-      # to loop through here until we find something that is the right type
-      # and doesn't break
-      document.xpath('//script[@type="application/ld+json"]').filter_map do |element|
-        json_ld = element.content&.gsub(CDATA_JUNK_PATTERN, '')
+    # Some publications have more than one JSON-LD definition on the page,
+    # and some of those definitions aren't valid JSON either, so we have
+    # to loop through here until we find something that is the right type
+    # and doesn't break
+    @structured_data ||= document.xpath('//script[@type="application/ld+json"]').filter_map do |element|
+      json_ld = element.content&.gsub(CDATA_JUNK_PATTERN, '')
 
-        next if json_ld.blank?
+      next if json_ld.blank?
 
-        structured_data = StructuredData.new(html_entities.decode(json_ld))
+      structured_data = StructuredData.new(html_entities.decode(json_ld))
 
-        next unless structured_data.valid?
+      next unless structured_data.valid?
 
-        structured_data
-      rescue Oj::ParseError, EncodingError
-        Rails.logger.debug("Invalid JSON-LD in #{@original_url}")
-        next
-      end.first
-    end
+      structured_data
+    rescue Oj::ParseError, EncodingError
+      Rails.logger.debug { "Invalid JSON-LD in #{@original_url}" }
+      next
+    end.first
   end
 
   def document
diff --git a/app/lib/ostatus/tag_manager.rb b/app/lib/ostatus/tag_manager.rb
index 4f4501312..7d8131622 100644
--- a/app/lib/ostatus/tag_manager.rb
+++ b/app/lib/ostatus/tag_manager.rb
@@ -5,27 +5,27 @@ class OStatus::TagManager
   include RoutingHelper
 
   VERBS = {
-    post:           'http://activitystrea.ms/schema/1.0/post',
-    share:          'http://activitystrea.ms/schema/1.0/share',
-    favorite:       'http://activitystrea.ms/schema/1.0/favorite',
-    unfavorite:     'http://activitystrea.ms/schema/1.0/unfavorite',
-    delete:         'http://activitystrea.ms/schema/1.0/delete',
-    follow:         'http://activitystrea.ms/schema/1.0/follow',
+    post: 'http://activitystrea.ms/schema/1.0/post',
+    share: 'http://activitystrea.ms/schema/1.0/share',
+    favorite: 'http://activitystrea.ms/schema/1.0/favorite',
+    unfavorite: 'http://activitystrea.ms/schema/1.0/unfavorite',
+    delete: 'http://activitystrea.ms/schema/1.0/delete',
+    follow: 'http://activitystrea.ms/schema/1.0/follow',
     request_friend: 'http://activitystrea.ms/schema/1.0/request-friend',
-    authorize:      'http://activitystrea.ms/schema/1.0/authorize',
-    reject:         'http://activitystrea.ms/schema/1.0/reject',
-    unfollow:       'http://ostatus.org/schema/1.0/unfollow',
-    block:          'http://mastodon.social/schema/1.0/block',
-    unblock:        'http://mastodon.social/schema/1.0/unblock',
+    authorize: 'http://activitystrea.ms/schema/1.0/authorize',
+    reject: 'http://activitystrea.ms/schema/1.0/reject',
+    unfollow: 'http://ostatus.org/schema/1.0/unfollow',
+    block: 'http://mastodon.social/schema/1.0/block',
+    unblock: 'http://mastodon.social/schema/1.0/unblock',
   }.freeze
 
   TYPES = {
-    activity:   'http://activitystrea.ms/schema/1.0/activity',
-    note:       'http://activitystrea.ms/schema/1.0/note',
-    comment:    'http://activitystrea.ms/schema/1.0/comment',
-    person:     'http://activitystrea.ms/schema/1.0/person',
+    activity: 'http://activitystrea.ms/schema/1.0/activity',
+    note: 'http://activitystrea.ms/schema/1.0/note',
+    comment: 'http://activitystrea.ms/schema/1.0/comment',
+    person: 'http://activitystrea.ms/schema/1.0/person',
     collection: 'http://activitystrea.ms/schema/1.0/collection',
-    group:      'http://activitystrea.ms/schema/1.0/group',
+    group: 'http://activitystrea.ms/schema/1.0/group',
   }.freeze
 
   COLLECTIONS = {
diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb
index 08aa29696..6fa2bc5d2 100644
--- a/app/lib/plain_text_formatter.rb
+++ b/app/lib/plain_text_formatter.rb
@@ -18,7 +18,7 @@ class PlainTextFormatter
     if local?
       text
     else
-      strip_tags(insert_newlines).chomp
+      html_entities.decode(strip_tags(insert_newlines)).chomp
     end
   end
 
@@ -27,4 +27,8 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
+
+  def html_entities
+    HTMLEntities.new
+  end
 end
diff --git a/app/lib/rate_limiter.rb b/app/lib/rate_limiter.rb
index 0e2c9a894..4a0b35b08 100644
--- a/app/lib/rate_limiter.rb
+++ b/app/lib/rate_limiter.rb
@@ -48,7 +48,7 @@ class RateLimiter
     {
       'X-RateLimit-Limit' => @limit.to_s,
       'X-RateLimit-Remaining' => (@limit - (redis.get(key) || 0).to_i).to_s,
-      'X-RateLimit-Reset' => (now + (@period - now.to_i % @period)).iso8601(6),
+      'X-RateLimit-Reset' => (now + (@period - (now.to_i % @period))).iso8601(6),
     }
   end
 
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 0508169dc..4bde6fc91 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -182,6 +182,7 @@ class Request
 
       contents = truncated_body(limit)
       raise Mastodon::LengthValidationError if contents.bytesize > limit
+
       contents
     end
   end
@@ -215,26 +216,24 @@ class Request
         addr_by_socket = {}
 
         addresses.each do |address|
-          begin
-            check_private_address(address, host)
+          check_private_address(address, host)
 
-            sock     = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
-            sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
+          sock     = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
+          sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
 
-            sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
+          sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
 
-            sock.connect_nonblock(sockaddr)
+          sock.connect_nonblock(sockaddr)
 
-            # If that hasn't raised an exception, we somehow managed to connect
-            # immediately, close pending sockets and return immediately
-            socks.each(&:close)
-            return sock
-          rescue IO::WaitWritable
-            socks << sock
-            addr_by_socket[sock] = sockaddr
-          rescue => e
-            outer_e = e
-          end
+          # If that hasn't raised an exception, we somehow managed to connect
+          # immediately, close pending sockets and return immediately
+          socks.each(&:close)
+          return sock
+        rescue IO::WaitWritable
+          socks << sock
+          addr_by_socket[sock] = sockaddr
+        rescue => e
+          outer_e = e
         end
 
         until socks.empty?
@@ -274,14 +273,14 @@ class Request
 
       def check_private_address(address, host)
         addr = IPAddr.new(address.to_s)
-        return if private_address_exceptions.any? { |range| range.include?(addr) }
+
+        return if Rails.env.development? || private_address_exceptions.any? { |range| range.include?(addr) }
+
         raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr)
       end
 
       def private_address_exceptions
-        @private_address_exceptions = begin
-          (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(',').map { |addr| IPAddr.new(addr) }
-        end
+        @private_address_exceptions = (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(',').map { |addr| IPAddr.new(addr) }
       end
     end
   end
diff --git a/app/lib/request_pool.rb b/app/lib/request_pool.rb
index e5899a79a..6be172286 100644
--- a/app/lib/request_pool.rb
+++ b/app/lib/request_pool.rb
@@ -64,7 +64,7 @@ class RequestPool
           retries     += 1
           retry
         end
-      rescue StandardError
+      rescue
         # If this connection raises errors of any kind, it's
         # better if it gets reaped as soon as possible
 
diff --git a/app/lib/scope_transformer.rb b/app/lib/scope_transformer.rb
index fdfc6cf13..adcb711f8 100644
--- a/app/lib/scope_transformer.rb
+++ b/app/lib/scope_transformer.rb
@@ -28,7 +28,7 @@ class ScopeTransformer < Parslet::Transform
     def merge!(other_scope)
       raise ArgumentError unless other_scope.namespace == namespace && other_scope.term == term
 
-      @access.concat(other_scope.instance_variable_get('@access'))
+      @access.concat(other_scope.instance_variable_get(:@access))
       @access.uniq!
       @access.sort!
 
diff --git a/app/lib/settings/scoped_settings.rb b/app/lib/settings/scoped_settings.rb
index 796de1113..983865a68 100644
--- a/app/lib/settings/scoped_settings.rb
+++ b/app/lib/settings/scoped_settings.rb
@@ -35,6 +35,7 @@ module Settings
 
       Setting.default_settings.each do |key, default_value|
         next if records.key?(key) || default_value.is_a?(Hash)
+
         records[key] = Setting.new(var: key, value: default_value)
       end
 
@@ -55,6 +56,7 @@ module Settings
         if db_val
           default_value = ScopedSettings.default_settings[key]
           return default_value.with_indifferent_access.merge!(db_val.value) if default_value.is_a?(Hash)
+
           db_val.value
         else
           ScopedSettings.default_settings[key]
diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb
index b6c80b801..c0e6f3331 100644
--- a/app/lib/status_filter.rb
+++ b/app/lib/status_filter.rb
@@ -11,6 +11,7 @@ class StatusFilter
 
   def filtered?
     return false if !account.nil? && account.id == status.account_id
+
     blocked_by_policy? || (account_present? && filtered_status?) || silenced_account?
   end
 
diff --git a/app/lib/status_finder.rb b/app/lib/status_finder.rb
index 22ced8bf8..1a7f2fe69 100644
--- a/app/lib/status_finder.rb
+++ b/app/lib/status_finder.rb
@@ -27,8 +27,6 @@ class StatusFinder
   end
 
   def verify_action!
-    unless recognized_params[:action] == 'show'
-      raise ActiveRecord::RecordNotFound
-    end
+    raise ActiveRecord::RecordNotFound unless recognized_params[:action] == 'show'
   end
 end
diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index a1d12a654..7fbf4437d 100644
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -25,6 +25,7 @@ class TagManager
   def local_url?(url)
     uri    = Addressable::URI.parse(url).normalize
     return false unless uri.host
+
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
 
     TagManager.instance.web_domain?(domain)
diff --git a/app/lib/themes.rb b/app/lib/themes.rb
index 81e016d4a..45ba47780 100644
--- a/app/lib/themes.rb
+++ b/app/lib/themes.rb
@@ -7,24 +7,23 @@ class Themes
   include Singleton
 
   def initialize
-
     core = YAML.load_file(Rails.root.join('app', 'javascript', 'core', 'theme.yml'))
-    core['pack'] = Hash.new unless core['pack']
+    core['pack'] = {} unless core['pack']
 
-    result = Hash.new
-    Dir.glob(Rails.root.join('app', 'javascript', 'flavours', '*', 'theme.yml')) do |path|
-      data = YAML.load_file(path)
+    result = {}
+    Rails.root.glob('app/javascript/flavours/*/theme.yml') do |pathname|
+      data = YAML.load_file(pathname)
       next unless data['pack']
 
-      dir = File.dirname(path)
-      name = File.basename(dir)
+      dir = pathname.dirname
+      name = dir.basename.to_s
       locales = []
       screenshots = []
 
       if data['locales']
         Dir.glob(File.join(dir, data['locales'], '*.{js,json}')) do |locale|
-          localeName = File.basename(locale, File.extname(locale))
-          locales.push(localeName) unless localeName.match(/defaultMessages|whitelist|index/)
+          locale_name = File.basename(locale, File.extname(locale))
+          locales.push(locale_name) unless /defaultMessages|whitelist|index/.match?(locale_name)
         end
       end
 
@@ -43,34 +42,30 @@ class Themes
       result[name] = data
     end
 
-    Dir.glob(Rails.root.join('app', 'javascript', 'skins', '*', '*')) do |path|
-      ext = File.extname(path)
-      skin = File.basename(path)
-      name = File.basename(File.dirname(path))
+    Rails.root.glob('app/javascript/skins/*/*') do |pathname|
+      ext = pathname.extname.to_s
+      skin = pathname.basename.to_s
+      name = pathname.dirname.basename.to_s
       next unless result[name]
 
-      if File.directory?(path)
+      if pathname.directory?
         pack = []
-        Dir.glob(File.join(path, '*.{css,scss}')) do |sheet|
-          pack.push(File.basename(sheet, File.extname(sheet)))
+        pathname.glob('*.{css,scss}') do |sheet|
+          pack.push(sheet.basename(sheet.extname).to_s)
         end
-      elsif ext.match(/^\.s?css$/i)
-        skin = File.basename(path, ext)
+      elsif /^\.s?css$/i.match?(ext)
+        skin = pathname.basename(ext).to_s
         pack = ['common']
       end
 
-      if skin != 'default'
-        result[name]['skin'][skin] = pack
-      end
+      result[name]['skin'][skin] = pack if skin != 'default'
     end
 
     @core = core
     @conf = result
   end
 
-  def core
-    @core
-  end
+  attr_reader :core
 
   def flavour(name)
     @conf[name]
@@ -86,7 +81,7 @@ class Themes
 
   def flavours_and_skins
     flavours.map do |flavour|
-      [flavour, skins_for(flavour).map{ |skin| [flavour, skin] }]
+      [flavour, skins_for(flavour).map { |skin| [flavour, skin] }]
     end
   end
 end
diff --git a/app/lib/toc_generator.rb b/app/lib/toc_generator.rb
deleted file mode 100644
index 0c8f766ca..000000000
--- a/app/lib/toc_generator.rb
+++ /dev/null
@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-class TOCGenerator
-  TARGET_ELEMENTS = %w(h1 h2 h3 h4 h5 h6).freeze
-  LISTED_ELEMENTS = %w(h2 h3).freeze
-
-  class Section
-    attr_accessor :depth, :title, :children, :anchor
-
-    def initialize(depth, title, anchor)
-      @depth    = depth
-      @title    = title
-      @children = []
-      @anchor   = anchor
-    end
-
-    delegate :<<, to: :children
-  end
-
-  def initialize(source_html)
-    @source_html = source_html
-    @processed   = false
-    @target_html = ''
-    @headers     = []
-    @slugs       = Hash.new { |h, k| h[k] = 0 }
-  end
-
-  def html
-    parse_and_transform unless @processed
-    @target_html
-  end
-
-  def toc
-    parse_and_transform unless @processed
-    @headers
-  end
-
-  private
-
-  def parse_and_transform
-    return if @source_html.blank?
-
-    parsed_html = Nokogiri::HTML.fragment(@source_html)
-
-    parsed_html.traverse do |node|
-      next unless TARGET_ELEMENTS.include?(node.name)
-
-      anchor = node['id'] || node.text.parameterize.presence || 'sec'
-      @slugs[anchor] += 1
-      anchor = "#{anchor}-#{@slugs[anchor]}" if @slugs[anchor] > 1
-
-      node['id'] = anchor
-
-      next unless LISTED_ELEMENTS.include?(node.name)
-
-      depth          = node.name[1..-1]
-      latest_section = @headers.last
-
-      if latest_section.nil? || latest_section.depth >= depth
-        @headers << Section.new(depth, node.text, anchor)
-      else
-        latest_section << Section.new(depth, node.text, anchor)
-      end
-    end
-
-    @target_html = parsed_html.to_s
-    @processed   = true
-  end
-end
diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb
index 285f30939..bfe5de44f 100644
--- a/app/lib/translation_service.rb
+++ b/app/lib/translation_service.rb
@@ -21,6 +21,10 @@ class TranslationService
     ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
   end
 
+  def languages
+    {}
+  end
+
   def translate(_text, _source_language, _target_language)
     raise NotImplementedError
   end
diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb
index 537fd24c0..afcb7ecb2 100644
--- a/app/lib/translation_service/deepl.rb
+++ b/app/lib/translation_service/deepl.rb
@@ -11,33 +11,59 @@ class TranslationService::DeepL < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
+    request(:post, '/v2/translate', form: form) do |res|
+      transform_response(res.body_with_limit)
+    end
+  end
+
+  def languages
+    source_languages = [nil] + fetch_languages('source')
+
+    # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
+    # they are supported but not returned by the API.
+    target_languages = %w(en pt) + fetch_languages('target')
+
+    source_languages.index_with { |language| target_languages.without(nil, language) }
+  end
+
+  private
+
+  def fetch_languages(type)
+    request(:get, "/v2/languages?type=#{type}") do |res|
+      Oj.load(res.body_with_limit).map { |language| normalize_language(language['language']) }
+    end
+  end
+
+  def normalize_language(language)
+    subtags = language.split(/[_-]/)
+    subtags[0].downcase!
+    subtags[1]&.upcase!
+    subtags.join('-')
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{base_url}#{path}", **options)
+    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 456
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
-    req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}")
-    req
-  end
-
-  def endpoint_url
+  def base_url
     if @plan == 'free'
-      'https://api-free.deepl.com/v2/translate'
+      'https://api-free.deepl.com'
     else
-      'https://api.deepl.com/v2/translate'
+      'https://api.deepl.com'
     end
   end
 
diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb
index 4ebe21e45..8bb194a9c 100644
--- a/app/lib/translation_service/libre_translate.rb
+++ b/app/lib/translation_service/libre_translate.rb
@@ -9,29 +9,41 @@ class TranslationService::LibreTranslate < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    request(:post, '/translate', body: body) do |res|
+      transform_response(res.body_with_limit, source_language)
+    end
+  end
+
+  def languages
+    request(:get, '/languages') do |res|
+      languages = Oj.load(res.body_with_limit).to_h do |language|
+        [language['code'], language['targets'].without(language['code'])]
+      end
+      languages[nil] = languages.values.flatten.uniq.sort
+      languages
+    end
+  end
+
+  private
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
+    req.add_headers('Content-Type': 'application/json')
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 403
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit, source_language)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
-    req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true)
-    req.add_headers('Content-Type': 'application/json')
-    req
-  end
-
   def transform_response(str, source_language)
     json = Oj.load(str, mode: :strict)
 
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
deleted file mode 100644
index 260077a1c..000000000
--- a/app/lib/user_settings_decorator.rb
+++ /dev/null
@@ -1,180 +0,0 @@
-# frozen_string_literal: true
-
-class UserSettingsDecorator
-  attr_reader :user, :settings
-
-  def initialize(user)
-    @user = user
-  end
-
-  def update(settings)
-    @settings = settings
-    process_update
-  end
-
-  private
-
-  def process_update
-    user.settings['notification_emails'] = merged_notification_emails if change?('notification_emails')
-    user.settings['interactions']        = merged_interactions if change?('interactions')
-    user.settings['default_privacy']     = default_privacy_preference if change?('setting_default_privacy')
-    user.settings['default_sensitive']   = default_sensitive_preference if change?('setting_default_sensitive')
-    user.settings['default_language']    = default_language_preference if change?('setting_default_language')
-    user.settings['unfollow_modal']      = unfollow_modal_preference if change?('setting_unfollow_modal')
-    user.settings['boost_modal']         = boost_modal_preference if change?('setting_boost_modal')
-    user.settings['favourite_modal']     = favourite_modal_preference if change?('setting_favourite_modal')
-    user.settings['delete_modal']        = delete_modal_preference if change?('setting_delete_modal')
-    user.settings['auto_play_gif']       = auto_play_gif_preference if change?('setting_auto_play_gif')
-    user.settings['display_media']       = display_media_preference if change?('setting_display_media')
-    user.settings['expand_spoilers']     = expand_spoilers_preference if change?('setting_expand_spoilers')
-    user.settings['reduce_motion']       = reduce_motion_preference if change?('setting_reduce_motion')
-    user.settings['disable_swiping']     = disable_swiping_preference if change?('setting_disable_swiping')
-    user.settings['system_font_ui']      = system_font_ui_preference if change?('setting_system_font_ui')
-    user.settings['system_emoji_font']   = system_emoji_font_preference if change?('setting_system_emoji_font')
-    user.settings['noindex']             = noindex_preference if change?('setting_noindex')
-    user.settings['hide_followers_count'] = hide_followers_count_preference if change?('setting_hide_followers_count')
-    user.settings['flavour']             = flavour_preference if change?('setting_flavour')
-    user.settings['skin']                = skin_preference if change?('setting_skin')
-    user.settings['aggregate_reblogs']   = aggregate_reblogs_preference if change?('setting_aggregate_reblogs')
-    user.settings['show_application']    = show_application_preference if change?('setting_show_application')
-    user.settings['advanced_layout']     = advanced_layout_preference if change?('setting_advanced_layout')
-    user.settings['default_content_type']= default_content_type_preference if change?('setting_default_content_type')
-    user.settings['use_blurhash']        = use_blurhash_preference if change?('setting_use_blurhash')
-    user.settings['use_pending_items']   = use_pending_items_preference if change?('setting_use_pending_items')
-    user.settings['trends']              = trends_preference if change?('setting_trends')
-    user.settings['crop_images']         = crop_images_preference if change?('setting_crop_images')
-    user.settings['always_send_emails']  = always_send_emails_preference if change?('setting_always_send_emails')
-  end
-
-  def merged_notification_emails
-    user.settings['notification_emails'].merge coerced_settings('notification_emails').to_h
-  end
-
-  def merged_interactions
-    user.settings['interactions'].merge coerced_settings('interactions').to_h
-  end
-
-  def default_privacy_preference
-    settings['setting_default_privacy']
-  end
-
-  def default_sensitive_preference
-    boolean_cast_setting 'setting_default_sensitive'
-  end
-
-  def unfollow_modal_preference
-    boolean_cast_setting 'setting_unfollow_modal'
-  end
-
-  def boost_modal_preference
-    boolean_cast_setting 'setting_boost_modal'
-  end
-
-  def favourite_modal_preference
-    boolean_cast_setting 'setting_favourite_modal'
-  end
-
-  def delete_modal_preference
-    boolean_cast_setting 'setting_delete_modal'
-  end
-
-  def system_font_ui_preference
-    boolean_cast_setting 'setting_system_font_ui'
-  end
-
-  def system_emoji_font_preference
-    boolean_cast_setting 'setting_system_emoji_font'
-  end
-
-  def auto_play_gif_preference
-    boolean_cast_setting 'setting_auto_play_gif'
-  end
-
-  def display_media_preference
-    settings['setting_display_media']
-  end
-
-  def expand_spoilers_preference
-    boolean_cast_setting 'setting_expand_spoilers'
-  end
-
-  def reduce_motion_preference
-    boolean_cast_setting 'setting_reduce_motion'
-  end
-
-  def disable_swiping_preference
-    boolean_cast_setting 'setting_disable_swiping'
-  end
-
-  def noindex_preference
-    boolean_cast_setting 'setting_noindex'
-  end
-
-  def flavour_preference
-    settings['setting_flavour']
-  end
-
-  def skin_preference
-    settings['setting_skin']
-  end
-
-  def hide_followers_count_preference
-    boolean_cast_setting 'setting_hide_followers_count'
-  end
-
-  def show_application_preference
-    boolean_cast_setting 'setting_show_application'
-  end
-
-  def default_language_preference
-    settings['setting_default_language']
-  end
-
-  def aggregate_reblogs_preference
-    boolean_cast_setting 'setting_aggregate_reblogs'
-  end
-
-  def advanced_layout_preference
-    boolean_cast_setting 'setting_advanced_layout'
-  end
-
-  def default_content_type_preference
-    settings['setting_default_content_type']
-  end
-
-  def use_blurhash_preference
-    boolean_cast_setting 'setting_use_blurhash'
-  end
-
-  def use_pending_items_preference
-    boolean_cast_setting 'setting_use_pending_items'
-  end
-
-  def trends_preference
-    boolean_cast_setting 'setting_trends'
-  end
-
-  def crop_images_preference
-    boolean_cast_setting 'setting_crop_images'
-  end
-
-  def always_send_emails_preference
-    boolean_cast_setting 'setting_always_send_emails'
-  end
-
-  def boolean_cast_setting(key)
-    ActiveModel::Type::Boolean.new.cast(settings[key])
-  end
-
-  def coerced_settings(key)
-    coerce_values settings.fetch(key, {})
-  end
-
-  def coerce_values(params_hash)
-    params_hash.transform_values { |x| ActiveModel::Type::Boolean.new.cast(x) }
-  end
-
-  def change?(key)
-    !settings[key].nil?
-  end
-end
diff --git a/app/lib/user_settings_serializer.rb b/app/lib/user_settings_serializer.rb
new file mode 100644
index 000000000..10d1be04d
--- /dev/null
+++ b/app/lib/user_settings_serializer.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class UserSettingsSerializer
+  def self.load(value)
+    json = begin
+      if value.blank?
+        {}
+      else
+        Oj.load(value, symbol_keys: true)
+      end
+    end
+
+    UserSettings.new(json)
+  end
+
+  def self.dump(value)
+    Oj.dump(value.as_json)
+  end
+end
diff --git a/app/lib/vacuum/access_tokens_vacuum.rb b/app/lib/vacuum/access_tokens_vacuum.rb
index 4f3878027..7b91f74a5 100644
--- a/app/lib/vacuum/access_tokens_vacuum.rb
+++ b/app/lib/vacuum/access_tokens_vacuum.rb
@@ -9,10 +9,10 @@ class Vacuum::AccessTokensVacuum
   private
 
   def vacuum_revoked_access_tokens!
-    Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+    Doorkeeper::AccessToken.where.not(revoked_at: nil).where('revoked_at < NOW()').delete_all
   end
 
   def vacuum_revoked_access_grants!
-    Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+    Doorkeeper::AccessGrant.where.not(revoked_at: nil).where('revoked_at < NOW()').delete_all
   end
 end
diff --git a/app/lib/validation_error_formatter.rb b/app/lib/validation_error_formatter.rb
index 3f964f739..1d3e8955b 100644
--- a/app/lib/validation_error_formatter.rb
+++ b/app/lib/validation_error_formatter.rb
@@ -19,7 +19,7 @@ class ValidationErrorFormatter
       messages = errors.messages[attribute_name]
 
       h[@aliases[attribute_name] || attribute_name] = attribute_errors.map.with_index do |error, index|
-        { error: 'ERR_' + error[:error].to_s.upcase, description: messages[index] }
+        { error: "ERR_#{error[:error].to_s.upcase}", description: messages[index] }
       end
     end
 
diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb
index 7c0c10c33..ae8a3b1ea 100644
--- a/app/lib/webfinger.rb
+++ b/app/lib/webfinger.rb
@@ -57,6 +57,7 @@ class Webfinger
       if res.code == 200
         body = res.body_with_limit
         raise Webfinger::Error, "Request for #{@uri} returned empty response" if body.empty?
+
         body
       elsif res.code == 404 && use_fallback
         body_from_host_meta
@@ -99,7 +100,7 @@ class Webfinger
   end
 
   def standard_url
-    if @domain.end_with? ".onion"
+    if @domain.end_with? '.onion'
       "http://#{@domain}/.well-known/webfinger?resource=#{@uri}"
     else
       "https://#{@domain}/.well-known/webfinger?resource=#{@uri}"
@@ -107,7 +108,7 @@ class Webfinger
   end
 
   def host_meta_url
-    if @domain.end_with? ".onion"
+    if @domain.end_with? '.onion'
       "http://#{@domain}/.well-known/host-meta"
     else
       "https://#{@domain}/.well-known/host-meta"