Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `README.md`:
  Upstream changed their README, we have our own.
  Kept ours.
- `app/helpers/application_helper.rb`:
  Minor code style fix upstream, on a line that is different in glitch-soc
  due to the different theming system.
  Applied the code style fix to our own code.
- `app/views/settings/preferences/appearance/show.html.haml`:
  Code style fix on a line next to lines exclusive to glitch-soc.
  Applied upstream changes.
- `yarn.lock`:
  Upstream updated a dependency textually close to a glitch-soc-only
  dependency.
  Updated the dependency like upstream did.

7 files changed, 142 insertions, 31 deletions

diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb
index f512635ab..89dfcef9f 100644
--- a/app/lib/admin/system_check.rb
+++ b/app/lib/admin/system_check.rb
@@ -2,6 +2,7 @@
 
 class Admin::SystemCheck
   ACTIVE_CHECKS = [
+    Admin::SystemCheck::MediaPrivacyCheck,
     Admin::SystemCheck::DatabaseSchemaCheck,
     Admin::SystemCheck::SidekiqProcessCheck,
     Admin::SystemCheck::RulesCheck,
diff --git a/app/lib/admin/system_check/media_privacy_check.rb b/app/lib/admin/system_check/media_privacy_check.rb
new file mode 100644
index 000000000..1df05b120
--- /dev/null
+++ b/app/lib/admin/system_check/media_privacy_check.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck
+  include RoutingHelper
+
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
+  def pass?
+    check_media_uploads!
+    @failure_message.nil?
+  end
+
+  def message
+    Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true)
+  end
+
+  private
+
+  def check_media_uploads!
+    if Rails.configuration.x.use_s3
+      check_media_listing_inaccessible_s3!
+    else
+      check_media_listing_inaccessible!
+    end
+  end
+
+  def check_media_listing_inaccessible!
+    full_url = full_asset_url(media_attachment.file.url(:original, false))
+
+    # Check if we can list the uploaded file. If true, that's an error
+    directory_url = Addressable::URI.parse(full_url)
+    directory_url.query = nil
+    filename = directory_url.path.gsub(%r{.*/}, '')
+    directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/')
+    Request.new(:get, directory_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?(filename)
+        @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error
+        @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS'
+      end
+    end
+  rescue
+    nil
+  end
+
+  def check_media_listing_inaccessible_s3!
+    urls_to_check = []
+    paperclip_options = Paperclip::Attachment.default_options
+    s3_protocol = paperclip_options[:s3_protocol]
+    s3_host_alias = paperclip_options[:s3_host_alias]
+    s3_host_name  = paperclip_options[:s3_host_name]
+    bucket_name = paperclip_options.dig(:s3_credentials, :bucket)
+
+    urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present?
+    urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/"
+    urls_to_check.uniq.each do |full_url|
+      check_s3_listing!(full_url)
+      break if @failure_message.present?
+    end
+  rescue
+    nil
+  end
+
+  def check_s3_listing!(full_url)
+    bucket_url = Addressable::URI.parse(full_url)
+    bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original))
+    bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}"
+    Request.new(:get, bucket_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?('ListBucketResult')
+        @failure_message = :upload_check_privacy_error_object_storage
+        @failure_action  = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3'
+      end
+    end
+  end
+
+  def media_attachment
+    @media_attachment ||= begin
+      attachment = Account.representative.media_attachments.first
+      if attachment.present?
+        attachment.touch # rubocop:disable Rails/SkipsModelValidations
+        attachment
+      else
+        create_test_attachment!
+      end
+    end
+  end
+
+  def create_test_attachment!
+    Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file|
+      tmp_file.write(
+        Base64.decode64(
+          '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \
+          'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \
+          'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \
+          'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \
+          'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \
+          'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q=='
+        )
+      )
+      tmp_file.flush
+      Account.representative.media_attachments.create!(file: tmp_file)
+    end
+  end
+end
diff --git a/app/lib/admin/system_check/message.rb b/app/lib/admin/system_check/message.rb
index bfcad3bf3..ad8d4b607 100644
--- a/app/lib/admin/system_check/message.rb
+++ b/app/lib/admin/system_check/message.rb
@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::Message
-  attr_reader :key, :value, :action
+  attr_reader :key, :value, :action, :critical
 
-  def initialize(key, value = nil, action = nil)
-    @key    = key
-    @value  = value
-    @action = action
+  def initialize(key, value = nil, action = nil, critical = false)
+    @key      = key
+    @value    = value
+    @action   = action
+    @critical = critical
   end
 end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 85716f999..4bde6fc91 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -273,7 +273,9 @@ class Request
 
       def check_private_address(address, host)
         addr = IPAddr.new(address.to_s)
-        return if private_address_exceptions.any? { |range| range.include?(addr) }
+
+        return if Rails.env.development? || private_address_exceptions.any? { |range| range.include?(addr) }
+
         raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr)
       end
 
diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb
index 5ff93674a..bfe5de44f 100644
--- a/app/lib/translation_service.rb
+++ b/app/lib/translation_service.rb
@@ -21,8 +21,8 @@ class TranslationService
     ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
   end
 
-  def supported?(_source_language, _target_language)
-    false
+  def languages
+    {}
   end
 
   def translate(_text, _source_language, _target_language)
diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb
index deff95a1d..afcb7ecb2 100644
--- a/app/lib/translation_service/deepl.rb
+++ b/app/lib/translation_service/deepl.rb
@@ -17,25 +17,31 @@ class TranslationService::DeepL < TranslationService
     end
   end
 
-  def supported?(source_language, target_language)
-    source_language.in?(languages('source')) && target_language.in?(languages('target'))
+  def languages
+    source_languages = [nil] + fetch_languages('source')
+
+    # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
+    # they are supported but not returned by the API.
+    target_languages = %w(en pt) + fetch_languages('target')
+
+    source_languages.index_with { |language| target_languages.without(nil, language) }
   end
 
   private
 
-  def languages(type)
-    Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
-      request(:get, "/v2/languages?type=#{type}") do |res|
-        # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
-        # they are supported but not returned by the API.
-        extra = type == 'source' ? [nil] : %w(en pt)
-        languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
-
-        languages + extra
-      end
+  def fetch_languages(type)
+    request(:get, "/v2/languages?type=#{type}") do |res|
+      Oj.load(res.body_with_limit).map { |language| normalize_language(language['language']) }
     end
   end
 
+  def normalize_language(language)
+    subtags = language.split(/[_-]/)
+    subtags[0].downcase!
+    subtags[1]&.upcase!
+    subtags.join('-')
+  end
+
   def request(verb, path, **options)
     req = Request.new(verb, "#{base_url}#{path}", **options)
     req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb
index 743e4d77f..8bb194a9c 100644
--- a/app/lib/translation_service/libre_translate.rb
+++ b/app/lib/translation_service/libre_translate.rb
@@ -15,22 +15,18 @@ class TranslationService::LibreTranslate < TranslationService
     end
   end
 
-  def supported?(source_language, target_language)
-    languages.key?(source_language) && languages[source_language].include?(target_language)
-  end
-
-  private
-
   def languages
-    Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
-      request(:get, '/languages') do |res|
-        languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
-        languages[nil] = languages.values.flatten.uniq
-        languages
+    request(:get, '/languages') do |res|
+      languages = Oj.load(res.body_with_limit).to_h do |language|
+        [language['code'], language['targets'].without(language['code'])]
       end
+      languages[nil] = languages.values.flatten.uniq.sort
+      languages
     end
   end
 
+  private
+
   def request(verb, path, **options)
     req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
     req.add_headers('Content-Type': 'application/json')