Merge pull request #1848 from ClearlyClaire/glitch-soc/merge-upstream

Merge upstream changes

20 files changed, 391 insertions, 31 deletions

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 7ff06ea39..f4c67cccd 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -116,12 +116,12 @@ class ActivityPub::Activity
   def dereference_object!
     return unless @object.is_a?(String)
 
-    dereferencer = ActivityPub::Dereferencer.new(@object, permitted_origin: @account.uri, signature_account: signed_fetch_account)
+    dereferencer = ActivityPub::Dereferencer.new(@object, permitted_origin: @account.uri, signature_actor: signed_fetch_actor)
 
     @object = dereferencer.object unless dereferencer.object.nil?
   end
 
-  def signed_fetch_account
+  def signed_fetch_actor
     return Account.find(@options[:delivered_to_account_id]) if @options[:delivered_to_account_id].present?
 
     first_mentioned_local_account || first_local_follower
@@ -163,15 +163,15 @@ class ActivityPub::Activity
   end
 
   def followed_by_local_accounts?
-    @account.passive_relationships.exists? || @options[:relayed_through_account]&.passive_relationships&.exists?
+    @account.passive_relationships.exists? || (@options[:relayed_through_actor].is_a?(Account) && @options[:relayed_through_actor].passive_relationships&.exists?)
   end
 
   def requested_through_relay?
-    @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled?
+    @options[:relayed_through_actor] && Relay.find_by(inbox_url: @options[:relayed_through_actor].inbox_url)&.enabled?
   end
 
   def reject_payload!
-    Rails.logger.info("Rejected #{@json['type']} activity #{@json['id']} from #{@account.uri}#{@options[:relayed_through_account] && "via #{@options[:relayed_through_account].uri}"}")
+    Rails.logger.info("Rejected #{@json['type']} activity #{@json['id']} from #{@account.uri}#{@options[:relayed_through_actor] && "via #{@options[:relayed_through_actor].uri}"}")
     nil
   end
 end
diff --git a/app/lib/activitypub/dereferencer.rb b/app/lib/activitypub/dereferencer.rb
index bea69608f..4d7756d71 100644
--- a/app/lib/activitypub/dereferencer.rb
+++ b/app/lib/activitypub/dereferencer.rb
@@ -3,10 +3,10 @@
 class ActivityPub::Dereferencer
   include JsonLdHelper
 
-  def initialize(uri, permitted_origin: nil, signature_account: nil)
+  def initialize(uri, permitted_origin: nil, signature_actor: nil)
     @uri               = uri
     @permitted_origin  = permitted_origin
-    @signature_account = signature_account
+    @signature_actor = signature_actor
   end
 
   def object
@@ -46,7 +46,7 @@ class ActivityPub::Dereferencer
 
     req.add_headers('Accept' => 'application/activity+json, application/ld+json')
     req.add_headers(headers) if headers
-    req.on_behalf_of(@signature_account) if @signature_account
+    req.on_behalf_of(@signature_actor) if @signature_actor
 
     req.perform do |res|
       if res.code == 200
diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb
index e853a970e..f90adaf6c 100644
--- a/app/lib/activitypub/linked_data_signature.rb
+++ b/app/lib/activitypub/linked_data_signature.rb
@@ -9,7 +9,7 @@ class ActivityPub::LinkedDataSignature
     @json = json.with_indifferent_access
   end
 
-  def verify_account!
+  def verify_actor!
     return unless @json['signature'].is_a?(Hash)
 
     type        = @json['signature']['type']
@@ -18,7 +18,7 @@ class ActivityPub::LinkedDataSignature
 
     return unless type == 'RsaSignature2017'
 
-    creator   = ActivityPub::TagManager.instance.uri_to_resource(creator_uri, Account)
+    creator   = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
     creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false)
 
     return if creator.nil?
@@ -35,7 +35,7 @@ class ActivityPub::LinkedDataSignature
   def sign!(creator, sign_with: nil)
     options = {
       'type'    => 'RsaSignature2017',
-      'creator' => [ActivityPub::TagManager.instance.uri_for(creator), '#main-key'].join,
+      'creator' => ActivityPub::TagManager.instance.key_uri_for(creator),
       'created' => Time.now.utc.iso8601,
     }
 
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index f6b9741fa..3d6b28ef5 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -44,6 +44,10 @@ class ActivityPub::TagManager
     end
   end
 
+  def key_uri_for(target)
+    [uri_for(target), '#main-key'].join
+  end
+
   def uri_for_username(username)
     account_url(username: username)
   end
@@ -155,6 +159,10 @@ class ActivityPub::TagManager
     path_params[param]
   end
 
+  def uri_to_actor(uri)
+    uri_to_resource(uri, Account)
+  end
+
   def uri_to_resource(uri, klass)
     return if uri.nil?
 
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index c607223fc..0bc7e254e 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -403,6 +403,7 @@ class FeedManager
   def filter_from_home?(status, receiver_id, crutches)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
+    return true  if crutches[:languages][status.account_id].present? && status.language.present? && !crutches[:languages][status.account_id].include?(status.language)
 
     check_for_blocks = crutches[:active_mentions][status.id] || []
     check_for_blocks.concat([status.account_id])
@@ -600,6 +601,7 @@ class FeedManager
     end
 
     crutches[:following]       = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).index_with(true)
+    crutches[:languages]       = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:account_id)).pluck(:target_account_id, :languages).to_h
     crutches[:hiding_reblogs]  = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).index_with(true)
     crutches[:blocking]        = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
     crutches[:muting]          = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
diff --git a/app/lib/redis_configuration.rb b/app/lib/redis_configuration.rb
index e14d6c8b6..f0e86d985 100644
--- a/app/lib/redis_configuration.rb
+++ b/app/lib/redis_configuration.rb
@@ -7,9 +7,7 @@ class RedisConfiguration
       @pool = ConnectionPool.new(size: new_pool_size) { new.connection }
     end
 
-    def with
-      pool.with { |redis| yield redis }
-    end
+    delegate :with, to: :pool
 
     def pool
       @pool ||= establish_pool(pool_size)
@@ -17,7 +15,7 @@ class RedisConfiguration
 
     def pool_size
       if Sidekiq.server?
-        Sidekiq.options[:concurrency]
+        Sidekiq[:concurrency]
       else
         ENV['MAX_THREADS'] || 5
       end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index f5123d776..648aa3085 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -40,12 +40,11 @@ class Request
     set_digest! if options.key?(:body)
   end
 
-  def on_behalf_of(account, key_id_format = :uri, sign_with: nil)
-    raise ArgumentError, 'account must not be nil' if account.nil?
+  def on_behalf_of(actor, sign_with: nil)
+    raise ArgumentError, 'actor must not be nil' if actor.nil?
 
-    @account       = account
-    @keypair       = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
-    @key_id_format = key_id_format
+    @actor         = actor
+    @keypair       = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @actor.keypair
 
     self
   end
@@ -79,7 +78,7 @@ class Request
   end
 
   def headers
-    (@account ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET)
+    (@actor ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET)
   end
 
   class << self
@@ -128,12 +127,7 @@ class Request
   end
 
   def key_id
-    case @key_id_format
-    when :acct
-      @account.to_webfinger_s
-    when :uri
-      [ActivityPub::TagManager.instance.uri_for(@account), '#main-key'].join
-    end
+    ActivityPub::TagManager.instance.key_uri_for(@actor)
   end
 
   def http_client
@@ -208,7 +202,7 @@ class Request
 
         addresses.each do |address|
           begin
-            check_private_address(address)
+            check_private_address(address, host)
 
             sock     = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
             sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
@@ -264,10 +258,10 @@ class Request
 
       alias new open
 
-      def check_private_address(address)
+      def check_private_address(address, host)
         addr = IPAddr.new(address.to_s)
         return if private_address_exceptions.any? { |range| range.include?(addr) }
-        raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(addr)
+        raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr)
       end
 
       def private_address_exceptions
diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb
new file mode 100644
index 000000000..526e26ae5
--- /dev/null
+++ b/app/lib/translation_service.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class TranslationService
+  class Error < StandardError; end
+  class NotConfiguredError < Error; end
+  class TooManyRequestsError < Error; end
+  class QuotaExceededError < Error; end
+  class UnexpectedResponseError < Error; end
+
+  def self.configured
+    if ENV['DEEPL_API_KEY'].present?
+      TranslationService::DeepL.new(ENV.fetch('DEEPL_PLAN', 'free'), ENV['DEEPL_API_KEY'])
+    elsif ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
+      TranslationService::LibreTranslate.new(ENV['LIBRE_TRANSLATE_ENDPOINT'], ENV['LIBRE_TRANSLATE_API_KEY'])
+    else
+      raise NotConfiguredError
+    end
+  end
+
+  def translate(_text, _source_language, _target_language)
+    raise NotImplementedError
+  end
+end
diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb
new file mode 100644
index 000000000..b75b604a8
--- /dev/null
+++ b/app/lib/translation_service/deepl.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class TranslationService::DeepL < TranslationService
+  include JsonLdHelper
+
+  def initialize(plan, api_key)
+    super()
+
+    @plan    = plan
+    @api_key = api_key
+  end
+
+  def translate(text, source_language, target_language)
+    request(text, source_language, target_language).perform do |res|
+      case res.code
+      when 429
+        raise TooManyRequestsError
+      when 456
+        raise QuotaExceededError
+      when 200...300
+        transform_response(res.body_with_limit)
+      else
+        raise UnexpectedResponseError
+      end
+    end
+  end
+
+  private
+
+  def request(text, source_language, target_language)
+    req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
+    req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}")
+    req
+  end
+
+  def endpoint_url
+    if @plan == 'free'
+      'https://api-free.deepl.com/v2/translate'
+    else
+      'https://api.deepl.com/v2/translate'
+    end
+  end
+
+  def transform_response(str)
+    json = Oj.load(str, mode: :strict)
+
+    raise UnexpectedResponseError unless json.is_a?(Hash)
+
+    Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase)
+  rescue Oj::ParseError
+    raise UnexpectedResponseError
+  end
+end
diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb
new file mode 100644
index 000000000..8cf26f868
--- /dev/null
+++ b/app/lib/translation_service/libre_translate.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class TranslationService::LibreTranslate < TranslationService
+  def initialize(base_url, api_key)
+    super()
+
+    @base_url = base_url
+    @api_key  = api_key
+  end
+
+  def translate(text, source_language, target_language)
+    request(text, source_language, target_language).perform do |res|
+      case res.code
+      when 429
+        raise TooManyRequestsError
+      when 403
+        raise QuotaExceededError
+      when 200...300
+        transform_response(res.body_with_limit, source_language)
+      else
+        raise UnexpectedResponseError
+      end
+    end
+  end
+
+  private
+
+  def request(text, source_language, target_language)
+    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    req = Request.new(:post, "#{@base_url}/translate", body: body)
+    req.add_headers('Content-Type': 'application/json')
+    req
+  end
+
+  def transform_response(str, source_language)
+    json = Oj.load(str, mode: :strict)
+
+    raise UnexpectedResponseError unless json.is_a?(Hash)
+
+    Translation.new(text: json['translatedText'], detected_source_language: source_language)
+  rescue Oj::ParseError
+    raise UnexpectedResponseError
+  end
+end
diff --git a/app/lib/translation_service/translation.rb b/app/lib/translation_service/translation.rb
new file mode 100644
index 000000000..a55b82574
--- /dev/null
+++ b/app/lib/translation_service/translation.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class TranslationService::Translation < ActiveModelSerializers::Model
+  attributes :text, :detected_source_language
+end
diff --git a/app/lib/vacuum.rb b/app/lib/vacuum.rb
new file mode 100644
index 000000000..9db1ec90b
--- /dev/null
+++ b/app/lib/vacuum.rb
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+module Vacuum; end
diff --git a/app/lib/vacuum/access_tokens_vacuum.rb b/app/lib/vacuum/access_tokens_vacuum.rb
new file mode 100644
index 000000000..4f3878027
--- /dev/null
+++ b/app/lib/vacuum/access_tokens_vacuum.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Vacuum::AccessTokensVacuum
+  def perform
+    vacuum_revoked_access_tokens!
+    vacuum_revoked_access_grants!
+  end
+
+  private
+
+  def vacuum_revoked_access_tokens!
+    Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+  end
+
+  def vacuum_revoked_access_grants!
+    Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+  end
+end
diff --git a/app/lib/vacuum/backups_vacuum.rb b/app/lib/vacuum/backups_vacuum.rb
new file mode 100644
index 000000000..3b83072f3
--- /dev/null
+++ b/app/lib/vacuum/backups_vacuum.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class Vacuum::BackupsVacuum
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_expired_backups! if retention_period?
+  end
+
+  private
+
+  def vacuum_expired_backups!
+    backups_past_retention_period.in_batches.destroy_all
+  end
+
+  def backups_past_retention_period
+    Backup.unscoped.where(Backup.arel_table[:created_at].lt(@retention_period.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb
new file mode 100644
index 000000000..00b9fd646
--- /dev/null
+++ b/app/lib/vacuum/feeds_vacuum.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Vacuum::FeedsVacuum
+  def perform
+    vacuum_inactive_home_feeds!
+    vacuum_inactive_list_feeds!
+    vacuum_inactive_direct_feeds!
+  end
+
+  private
+
+  def vacuum_inactive_home_feeds!
+    inactive_users.select(:id, :account_id).find_in_batches do |users|
+      feed_manager.clean_feeds!(:home, users.map(&:account_id))
+    end
+  end
+
+  def vacuum_inactive_list_feeds!
+    inactive_users_lists.select(:id).find_in_batches do |lists|
+      feed_manager.clean_feeds!(:list, lists.map(&:id))
+    end
+  end
+
+  def vacuum_inactive_direct_feeds!
+    inactive_users_lists.select(:id).find_in_batches do |lists|
+      feed_manager.clean_feeds!(:direct, lists.map(&:id))
+    end
+  end
+
+  def inactive_users
+    User.confirmed.inactive
+  end
+
+  def inactive_users_lists
+    List.where(account_id: inactive_users.select(:account_id))
+  end
+
+  def feed_manager
+    FeedManager.instance
+  end
+end
diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb
new file mode 100644
index 000000000..7fb347ce4
--- /dev/null
+++ b/app/lib/vacuum/media_attachments_vacuum.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Vacuum::MediaAttachmentsVacuum
+  TTL = 1.day.freeze
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_cached_files! if retention_period?
+    vacuum_orphaned_records!
+  end
+
+  private
+
+  def vacuum_cached_files!
+    media_attachments_past_retention_period.find_each do |media_attachment|
+      media_attachment.file.destroy
+      media_attachment.thumbnail.destroy
+      media_attachment.save
+    end
+  end
+
+  def vacuum_orphaned_records!
+    orphaned_media_attachments.in_batches.destroy_all
+  end
+
+  def media_attachments_past_retention_period
+    MediaAttachment.unscoped.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago))
+  end
+
+  def orphaned_media_attachments
+    MediaAttachment.unscoped.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb
new file mode 100644
index 000000000..84ef100ed
--- /dev/null
+++ b/app/lib/vacuum/preview_cards_vacuum.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Vacuum::PreviewCardsVacuum
+  TTL = 1.day.freeze
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_cached_images! if retention_period?
+    vacuum_orphaned_records!
+  end
+
+  private
+
+  def vacuum_cached_images!
+    preview_cards_past_retention_period.find_each do |preview_card|
+      preview_card.image.destroy
+      preview_card.save
+    end
+  end
+
+  def vacuum_orphaned_records!
+    orphaned_preview_cards.in_batches.destroy_all
+  end
+
+  def preview_cards_past_retention_period
+    PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago))
+  end
+
+  def orphaned_preview_cards
+    PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb
new file mode 100644
index 000000000..41d6ba270
--- /dev/null
+++ b/app/lib/vacuum/statuses_vacuum.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class Vacuum::StatusesVacuum
+  include Redisable
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_statuses! if retention_period?
+  end
+
+  private
+
+  def vacuum_statuses!
+    statuses_scope.find_in_batches do |statuses|
+      # Side-effects not covered by foreign keys, such
+      # as the search index, must be handled first.
+
+      remove_from_account_conversations(statuses)
+      remove_from_search_index(statuses)
+
+      # Foreign keys take care of most associated records
+      # for us. Media attachments will be orphaned.
+
+      Status.where(id: statuses.map(&:id)).delete_all
+    end
+  end
+
+  def statuses_scope
+    Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility)
+  end
+
+  def retention_period_as_id
+    Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false)
+  end
+
+  def analyze_statuses!
+    ActiveRecord::Base.connection.execute('ANALYZE statuses')
+  end
+
+  def remove_from_account_conversations(statuses)
+    Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations)
+  end
+
+  def remove_from_search_index(statuses)
+    with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled?
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/system_keys_vacuum.rb b/app/lib/vacuum/system_keys_vacuum.rb
new file mode 100644
index 000000000..ceee2fd16
--- /dev/null
+++ b/app/lib/vacuum/system_keys_vacuum.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Vacuum::SystemKeysVacuum
+  def perform
+    vacuum_expired_system_keys!
+  end
+
+  private
+
+  def vacuum_expired_system_keys!
+    SystemKey.expired.delete_all
+  end
+end
diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb
index a681e0815..7c0c10c33 100644
--- a/app/lib/webfinger.rb
+++ b/app/lib/webfinger.rb
@@ -3,7 +3,7 @@
 class Webfinger
   class Error < StandardError; end
   class GoneError < Error; end
-  class RedirectError < StandardError; end
+  class RedirectError < Error; end
 
   class Response
     attr_reader :uri