Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- `app/controllers/activitypub/collections_controller.rb`:
  Conflict due to glitch-soc having to take care of local-only
  pinned toots in that controller.
  Took upstream's changes and restored the local-only special
  handling.
- `app/controllers/auth/sessions_controller.rb`:
  Minor conflicts due to the theming system, applied upstream
  changes, adapted the following two files for glitch-soc's
  theming system:
  - `app/controllers/concerns/sign_in_token_authentication_concern.rb`
  - `app/controllers/concerns/two_factor_authentication_concern.rb`
- `app/services/backup_service.rb`:
  Minor conflict due to glitch-soc having to handle local-only
  toots specially. Applied upstream changes and restored
  the local-only special handling.
- `app/views/admin/custom_emojis/index.html.haml`:
  Minor conflict due to the theming system.
- `package.json`:
  Upstream dependency updated, too close to a glitch-soc-only
  dependency in the file.
- `yarn.lock`:
  Upstream dependency updated, too close to a glitch-soc-only
  dependency in the file.

12 files changed, 314 insertions, 55 deletions

diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 7b4c53d50..f4276cece 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -76,6 +76,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.shared_inbox_url        = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
     @account.followers_url           = @json['followers'] || ''
     @account.featured_collection_url = @json['featured'] || ''
+    @account.devices_url             = @json['devices'] || ''
     @account.url                     = url || @uri
     @account.uri                     = @uri
     @account.display_name            = @json['name'] || ''
diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 989fd6784..749c84736 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -22,7 +22,7 @@ class BackupService < BaseService
 
     account.statuses.with_includes.reorder(nil).find_in_batches do |statuses|
       statuses.each do |status|
-        item = serialize_payload(status, ActivityPub::ActivitySerializer, signer: @account, allow_local_only: true)
+        item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer, signer: @account, allow_local_only: true)
         item.delete(:'@context')
 
         unless item[:type] == 'Announce' || item[:object][:attachment].blank?
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 9f0860674..dc23ef8d8 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -26,59 +26,20 @@ class BlockDomainService < BaseService
       suspend_accounts!
     end
 
-    clear_media! if domain_block.reject_media?
-  end
-
-  def invalidate_association_caches!
-    # Normally, associated models of a status are immutable (except for accounts)
-    # so they are aggressively cached. After updating the media attachments to no
-    # longer point to a local file, we need to clear the cache to make those
-    # changes appear in the API and UI
-    @affected_status_ids.each { |id| Rails.cache.delete_matched("statuses/#{id}-*") }
+    DomainClearMediaWorker.perform_async(domain_block.id) if domain_block.reject_media?
   end
 
   def silence_accounts!
     blocked_domain_accounts.without_silenced.in_batches.update_all(silenced_at: @domain_block.created_at)
   end
 
-  def clear_media!
-    @affected_status_ids = []
-
-    clear_account_images!
-    clear_account_attachments!
-    clear_emojos!
-
-    invalidate_association_caches!
-  end
-
   def suspend_accounts!
-    blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
+    blocked_domain_accounts.without_suspended.in_batches.update_all(suspended_at: @domain_block.created_at)
+    blocked_domain_accounts.where(suspended_at: @domain_block.created_at).reorder(nil).find_each do |account|
       SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at)
     end
   end
 
-  def clear_account_images!
-    blocked_domain_accounts.reorder(nil).find_each do |account|
-      account.avatar.destroy if account.avatar.exists?
-      account.header.destroy if account.header.exists?
-      account.save
-    end
-  end
-
-  def clear_account_attachments!
-    media_from_blocked_domain.reorder(nil).find_each do |attachment|
-      @affected_status_ids << attachment.status_id if attachment.status_id.present?
-
-      attachment.file.destroy if attachment.file.exists?
-      attachment.type = :unknown
-      attachment.save
-    end
-  end
-
-  def clear_emojos!
-    emojis_from_blocked_domains.destroy_all
-  end
-
   def blocked_domain
     domain_block.domain
   end
@@ -86,12 +47,4 @@ class BlockDomainService < BaseService
   def blocked_domain_accounts
     Account.by_domain_and_subdomains(blocked_domain)
   end
-
-  def media_from_blocked_domain
-    MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
-  end
-
-  def emojis_from_blocked_domains
-    CustomEmoji.by_domain_and_subdomains(blocked_domain)
-  end
 end
diff --git a/app/services/clear_domain_media_service.rb b/app/services/clear_domain_media_service.rb
new file mode 100644
index 000000000..704cfb71a
--- /dev/null
+++ b/app/services/clear_domain_media_service.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+class ClearDomainMediaService < BaseService
+  attr_reader :domain_block
+
+  def call(domain_block)
+    @domain_block = domain_block
+    clear_media! if domain_block.reject_media?
+  end
+
+  private
+
+  def invalidate_association_caches!
+    # Normally, associated models of a status are immutable (except for accounts)
+    # so they are aggressively cached. After updating the media attachments to no
+    # longer point to a local file, we need to clear the cache to make those
+    # changes appear in the API and UI
+    @affected_status_ids.each { |id| Rails.cache.delete_matched("statuses/#{id}-*") }
+  end
+
+  def clear_media!
+    @affected_status_ids = []
+
+    begin
+      clear_account_images!
+      clear_account_attachments!
+      clear_emojos!
+    ensure
+      invalidate_association_caches!
+    end
+  end
+
+  def clear_account_images!
+    blocked_domain_accounts.reorder(nil).find_each do |account|
+      account.avatar.destroy if account.avatar&.exists?
+      account.header.destroy if account.header&.exists?
+      account.save
+    end
+  end
+
+  def clear_account_attachments!
+    media_from_blocked_domain.reorder(nil).find_each do |attachment|
+      @affected_status_ids << attachment.status_id if attachment.status_id.present?
+
+      attachment.file.destroy if attachment.file&.exists?
+      attachment.type = :unknown
+      attachment.save
+    end
+  end
+
+  def clear_emojos!
+    emojis_from_blocked_domains.destroy_all
+  end
+
+  def blocked_domain
+    domain_block.domain
+  end
+
+  def blocked_domain_accounts
+    Account.by_domain_and_subdomains(blocked_domain)
+  end
+
+  def media_from_blocked_domain
+    MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
+  end
+
+  def emojis_from_blocked_domains
+    CustomEmoji.by_domain_and_subdomains(blocked_domain)
+  end
+end
diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb
index 7f9f21c4b..3e45570c3 100644
--- a/app/services/concerns/payloadable.rb
+++ b/app/services/concerns/payloadable.rb
@@ -5,8 +5,9 @@ module Payloadable
     signer    = options.delete(:signer)
     sign_with = options.delete(:sign_with)
     payload   = ActiveModelSerializers::SerializableResource.new(record, options.merge(serializer: serializer, adapter: ActivityPub::Adapter)).as_json
+    object    = record.respond_to?(:virtual_object) ? record.virtual_object : record
 
-    if (record.respond_to?(:sign?) && record.sign?) && signer && signing_enabled?
+    if (object.respond_to?(:sign?) && object.sign?) && signer && signing_enabled?
       ActivityPub::LinkedDataSignature.new(payload).sign!(signer, sign_with: sign_with)
     else
       payload
diff --git a/app/services/deliver_to_device_service.rb b/app/services/deliver_to_device_service.rb
new file mode 100644
index 000000000..71711945c
--- /dev/null
+++ b/app/services/deliver_to_device_service.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class DeliverToDeviceService < BaseService
+  include Payloadable
+
+  class EncryptedMessage < ActiveModelSerializers::Model
+    attributes :source_account, :target_account, :source_device,
+               :target_device_id, :type, :body, :digest,
+               :message_franking
+  end
+
+  def call(source_account, source_device, options = {})
+    @source_account   = source_account
+    @source_device    = source_device
+    @target_account   = Account.find(options[:account_id])
+    @target_device_id = options[:device_id]
+    @body             = options[:body]
+    @type             = options[:type]
+    @hmac             = options[:hmac]
+
+    set_message_franking!
+
+    if @target_account.local?
+      deliver_to_local!
+    else
+      deliver_to_remote!
+    end
+  end
+
+  private
+
+  def set_message_franking!
+    @message_franking = message_franking.to_token
+  end
+
+  def deliver_to_local!
+    target_device = @target_account.devices.find_by!(device_id: @target_device_id)
+
+    target_device.encrypted_messages.create!(
+      from_account: @source_account,
+      from_device_id: @source_device.device_id,
+      type: @type,
+      body: @body,
+      digest: @hmac,
+      message_franking: @message_franking
+    )
+  end
+
+  def deliver_to_remote!
+    ActivityPub::DeliveryWorker.perform_async(
+      Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_encrypted_message(encrypted_message), ActivityPub::ActivitySerializer)),
+      @source_account.id,
+      @target_account.inbox_url
+    )
+  end
+
+  def message_franking
+    MessageFranking.new(
+      source_account_id: @source_account.id,
+      target_account_id: @target_account.id,
+      hmac: @hmac,
+      timestamp: Time.now.utc
+    )
+  end
+
+  def encrypted_message
+    EncryptedMessage.new(
+      source_account: @source_account,
+      target_account: @target_account,
+      source_device: @source_device,
+      target_device_id: @target_device_id,
+      type: @type,
+      body: @body,
+      digest: @hmac,
+      message_franking: @message_franking
+    )
+  end
+end
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index c0d741d57..4cad93767 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -81,7 +81,9 @@ class ImportService < BaseService
       end
     end
 
-    Import::RelationshipWorker.push_bulk(items) do |acct, extra|
+    head_items = items.uniq { |acct, _| acct.split('@')[1] }
+    tail_items = items - head_items
+    Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
       [@account.id, acct, action, extra]
     end
   end
diff --git a/app/services/keys/claim_service.rb b/app/services/keys/claim_service.rb
new file mode 100644
index 000000000..672119130
--- /dev/null
+++ b/app/services/keys/claim_service.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+class Keys::ClaimService < BaseService
+  HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
+
+  class Result < ActiveModelSerializers::Model
+    attributes :account, :device_id, :key_id,
+               :key, :signature
+
+    def initialize(account, device_id, key_attributes = {})
+      @account   = account
+      @device_id = device_id
+      @key_id    = key_attributes[:key_id]
+      @key       = key_attributes[:key]
+      @signature = key_attributes[:signature]
+    end
+  end
+
+  def call(source_account, target_account_id, device_id)
+    @source_account = source_account
+    @target_account = Account.find(target_account_id)
+    @device_id      = device_id
+
+    if @target_account.local?
+      claim_local_key!
+    else
+      claim_remote_key!
+    end
+  rescue ActiveRecord::RecordNotFound
+    nil
+  end
+
+  private
+
+  def claim_local_key!
+    device = @target_account.devices.find_by(device_id: @device_id)
+    key    = nil
+
+    ApplicationRecord.transaction do
+      key = device.one_time_keys.order(Arel.sql('random()')).first!
+      key.destroy!
+    end
+
+    @result = Result.new(@target_account, @device_id, key)
+  end
+
+  def claim_remote_key!
+    query_result = QueryService.new.call(@target_account)
+    device       = query_result.find(@device_id)
+
+    return unless device.present? && device.valid_claim_url?
+
+    json = fetch_resource_with_post(device.claim_url)
+
+    return unless json.present? && json['publicKeyBase64'].present?
+
+    @result = Result.new(@target_account, @device_id, key_id: json['id'], key: json['publicKeyBase64'], signature: json.dig('signature', 'signatureValue'))
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
+    Rails.logger.debug "Claiming one-time key for #{@target_account.acct}:#{@device_id} failed: #{e}"
+    nil
+  end
+
+  def fetch_resource_with_post(uri)
+    build_post_request(uri).perform do |response|
+      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
+
+      body_to_json(response.body_with_limit) if response.code == 200
+    end
+  end
+
+  def build_post_request(uri)
+    Request.new(:post, uri).tap do |request|
+      request.on_behalf_of(@source_account, :uri)
+      request.add_headers(HEADERS)
+    end
+  end
+end
diff --git a/app/services/keys/query_service.rb b/app/services/keys/query_service.rb
new file mode 100644
index 000000000..286fbd834
--- /dev/null
+++ b/app/services/keys/query_service.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+class Keys::QueryService < BaseService
+  include JsonLdHelper
+
+  class Result < ActiveModelSerializers::Model
+    attributes :account, :devices
+
+    def initialize(account, devices)
+      @account = account
+      @devices = devices || []
+    end
+
+    def find(device_id)
+      @devices.find { |device| device.device_id == device_id }
+    end
+  end
+
+  class Device < ActiveModelSerializers::Model
+    attributes :device_id, :name, :identity_key, :fingerprint_key
+
+    def initialize(attributes = {})
+      @device_id       = attributes[:device_id]
+      @name            = attributes[:name]
+      @identity_key    = attributes[:identity_key]
+      @fingerprint_key = attributes[:fingerprint_key]
+      @claim_url       = attributes[:claim_url]
+    end
+
+    def valid_claim_url?
+      return false if @claim_url.blank?
+
+      begin
+        parsed_url = Addressable::URI.parse(@claim_url).normalize
+      rescue Addressable::URI::InvalidURIError
+        return false
+      end
+
+      %w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
+    end
+  end
+
+  def call(account)
+    @account = account
+
+    if @account.local?
+      query_local_devices!
+    else
+      query_remote_devices!
+    end
+
+    Result.new(@account, @devices)
+  end
+
+  private
+
+  def query_local_devices!
+    @devices = @account.devices.map { |device| Device.new(device) }
+  end
+
+  def query_remote_devices!
+    return if @account.devices_url.blank?
+
+    json = fetch_resource(@account.devices_url)
+
+    return if json['items'].blank?
+
+    @devices = json['items'].map do |device|
+      Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim'])
+    end
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
+    Rails.logger.debug "Querying devices for #{@account.acct} failed: #{e}"
+    nil
+  end
+end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 3c257451c..65a3f64b8 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -65,7 +65,7 @@ class ProcessMentionsService < BaseService
 
   def activitypub_json
     return @activitypub_json if defined?(@activitypub_json)
-    @activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
+    @activitypub_json = Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
   end
 
   def resolve_account_service
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 0a46509f8..6cecb5ac4 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -60,6 +60,6 @@ class ReblogService < BaseService
   end
 
   def build_json(reblog)
-    Oj.dump(serialize_payload(reblog, ActivityPub::ActivitySerializer, signer: reblog.account))
+    Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
   end
 end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 17ace100c..ba77552c6 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -112,6 +112,8 @@ class ResolveAccountService < BaseService
   end
 
   def webfinger_update_due?
+    return false if @options[:check_delivery_availability] && !DeliveryFailureTracker.available?(@domain)
+
     @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
   end