Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `README.md`:
  Upstream README has been changed, but we have a completely different one.
  Kept our `README.md`.
- `lib/sanitize_ext/sanitize_config.rb`:
  Upstream added support for more incoming HTML tags (a large subset of what
  glitch-soc accepts).
  Change the code style to match upstream's but otherwise do not change our
  code.
- `spec/lib/sanitize_config_spec.rb`:
  Upstream added support for more incoming HTML tags (a large subset of what
  glitch-soc accepts).
  Kept our version, since the tests are mostly glitch-soc's, except for cases
  which are purposefuly different.

4 files changed, 74 insertions, 34 deletions

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index be5b68b3f..4ce888fc9 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -322,27 +322,27 @@ class FeedManager
   def clean_feeds!(type, ids)
     reblogged_id_sets = {}
 
-    redis.pipelined do
+    redis.pipelined do |pipeline|
       ids.each do |feed_id|
-        redis.del(key(type, feed_id))
         reblog_key = key(type, feed_id, 'reblogs')
         # We collect a future for this: we don't block while getting
         # it, but we can iterate over it later.
-        reblogged_id_sets[feed_id] = redis.zrange(reblog_key, 0, -1)
-        redis.del(reblog_key)
+        reblogged_id_sets[feed_id] = pipeline.zrange(reblog_key, 0, -1)
+        pipeline.del(key(type, feed_id), reblog_key)
       end
     end
 
     # Remove all of the reblog tracking keys we just removed the
     # references to.
-    redis.pipelined do
-      reblogged_id_sets.each do |feed_id, future|
-        future.value.each do |reblogged_id|
-          reblog_set_key = key(type, feed_id, "reblogs:#{reblogged_id}")
-          redis.del(reblog_set_key)
-        end
+    keys_to_delete = reblogged_id_sets.flat_map do |feed_id, future|
+      future.value.map do |reblogged_id|
+        key(type, feed_id, "reblogs:#{reblogged_id}")
       end
     end
+
+    redis.del(keys_to_delete) unless keys_to_delete.empty?
+
+    nil
   end
 
   private
diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb
index 285f30939..5ff93674a 100644
--- a/app/lib/translation_service.rb
+++ b/app/lib/translation_service.rb
@@ -21,6 +21,10 @@ class TranslationService
     ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
   end
 
+  def supported?(_source_language, _target_language)
+    false
+  end
+
   def translate(_text, _source_language, _target_language)
     raise NotImplementedError
   end
diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb
index 151d33d90..deff95a1d 100644
--- a/app/lib/translation_service/deepl.rb
+++ b/app/lib/translation_service/deepl.rb
@@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
+    request(:post, '/v2/translate', form: form) do |res|
+      transform_response(res.body_with_limit)
+    end
+  end
+
+  def supported?(source_language, target_language)
+    source_language.in?(languages('source')) && target_language.in?(languages('target'))
+  end
+
+  private
+
+  def languages(type)
+    Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
+      request(:get, "/v2/languages?type=#{type}") do |res|
+        # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
+        # they are supported but not returned by the API.
+        extra = type == 'source' ? [nil] : %w(en pt)
+        languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
+
+        languages + extra
+      end
+    end
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{base_url}#{path}", **options)
+    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 456
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
-    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
-    req
-  end
-
-  def endpoint_url
+  def base_url
     if @plan == 'free'
-      'https://api-free.deepl.com/v2/translate'
+      'https://api-free.deepl.com'
     else
-      'https://api.deepl.com/v2/translate'
+      'https://api.deepl.com'
     end
   end
 
diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb
index 4ebe21e45..743e4d77f 100644
--- a/app/lib/translation_service/libre_translate.rb
+++ b/app/lib/translation_service/libre_translate.rb
@@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    request(:post, '/translate', body: body) do |res|
+      transform_response(res.body_with_limit, source_language)
+    end
+  end
+
+  def supported?(source_language, target_language)
+    languages.key?(source_language) && languages[source_language].include?(target_language)
+  end
+
+  private
+
+  def languages
+    Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
+      request(:get, '/languages') do |res|
+        languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
+        languages[nil] = languages.values.flatten.uniq
+        languages
+      end
+    end
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
+    req.add_headers('Content-Type': 'application/json')
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 403
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit, source_language)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
-    req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true)
-    req.add_headers('Content-Type': 'application/json')
-    req
-  end
-
   def transform_response(str, source_language)
     json = Oj.load(str, mode: :strict)