From 5d8398c8b8b51ee7363e7d45acc560f489783e34 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 2 Jun 2020 19:24:53 +0200
Subject: Add E2EE API (#13820)

---
 app/workers/activitypub/distribution_worker.rb        |  2 +-
 app/workers/activitypub/reply_distribution_worker.rb  |  2 +-
 app/workers/push_conversation_worker.rb               |  3 ++-
 app/workers/push_encrypted_message_worker.rb          | 16 ++++++++++++++++
 app/workers/scheduler/doorkeeper_cleanup_scheduler.rb |  1 +
 5 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 app/workers/push_encrypted_message_worker.rb

(limited to 'app/workers')

diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb
index 11b6a6111..e4997ba0e 100644
--- a/app/workers/activitypub/distribution_worker.rb
+++ b/app/workers/activitypub/distribution_worker.rb
@@ -43,7 +43,7 @@ class ActivityPub::DistributionWorker
   end
 
   def payload
-    @payload ||= Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @account))
+    @payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @account))
   end
 
   def relay!
diff --git a/app/workers/activitypub/reply_distribution_worker.rb b/app/workers/activitypub/reply_distribution_worker.rb
index 1ff8a657e..d4d0148ac 100644
--- a/app/workers/activitypub/reply_distribution_worker.rb
+++ b/app/workers/activitypub/reply_distribution_worker.rb
@@ -29,6 +29,6 @@ class ActivityPub::ReplyDistributionWorker
   end
 
   def payload
-    @payload ||= Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
+    @payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
   end
 end
diff --git a/app/workers/push_conversation_worker.rb b/app/workers/push_conversation_worker.rb
index 16f538215..aa858f715 100644
--- a/app/workers/push_conversation_worker.rb
+++ b/app/workers/push_conversation_worker.rb
@@ -2,13 +2,14 @@
 
 class PushConversationWorker
   include Sidekiq::Worker
+  include Redisable
 
   def perform(conversation_account_id)
     conversation = AccountConversation.find(conversation_account_id)
     message      = InlineRenderer.render(conversation, conversation.account, :conversation)
     timeline_id  = "timeline:direct:#{conversation.account_id}"
 
-    Redis.current.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
+    redis.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/app/workers/push_encrypted_message_worker.rb b/app/workers/push_encrypted_message_worker.rb
new file mode 100644
index 000000000..031230172
--- /dev/null
+++ b/app/workers/push_encrypted_message_worker.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class PushEncryptedMessageWorker
+  include Sidekiq::Worker
+  include Redisable
+
+  def perform(encrypted_message_id)
+    encrypted_message = EncryptedMessage.find(encrypted_message_id)
+    message           = InlineRenderer.render(encrypted_message, nil, :encrypted_message)
+    timeline_id       = "timeline:#{encrypted_message.device.account_id}:#{encrypted_message.device.device_id}"
+
+    redis.publish(timeline_id, Oj.dump(event: :encrypted_message, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
index 94788a85b..bb9dd49ca 100644
--- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
+++ b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
@@ -8,5 +8,6 @@ class Scheduler::DoorkeeperCleanupScheduler
   def perform
     Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
     Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+    SystemKey.expired.delete_all
   end
 end
-- 
cgit 


From d890abfcab076f5df4eafb75397435d32c641e20 Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Tue, 9 Jun 2020 17:26:58 +0900
Subject: Fix performance of follow import (#13836)

---
 app/helpers/webfinger_helper.rb           | 19 +++++++++++++++++++
 app/services/import_service.rb            |  4 +++-
 app/services/resolve_account_service.rb   |  2 ++
 app/workers/import/relationship_worker.rb | 21 ++++++++++++++++++++-
 4 files changed, 44 insertions(+), 2 deletions(-)

(limited to 'app/workers')

diff --git a/app/helpers/webfinger_helper.rb b/app/helpers/webfinger_helper.rb
index 70c493210..ab7ca4698 100644
--- a/app/helpers/webfinger_helper.rb
+++ b/app/helpers/webfinger_helper.rb
@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+# Monkey-patch on monkey-patch.
+# Because it conflicts with the request.rb patch.
+class HTTP::Timeout::PerOperationOriginal < HTTP::Timeout::PerOperation
+  def connect(socket_class, host, port, nodelay = false)
+    ::Timeout.timeout(@connect_timeout, HTTP::TimeoutError) do
+      @socket = socket_class.open(host, port)
+      @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
+    end
+  end
+end
+
 module WebfingerHelper
   def webfinger!(uri)
     hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri)
@@ -12,6 +23,14 @@ module WebfingerHelper
       headers: {
         'User-Agent': Mastodon::Version.user_agent,
       },
+
+      timeout_class: HTTP::Timeout::PerOperationOriginal,
+
+      timeout_options: {
+        write_timeout: 10,
+        connect_timeout: 5,
+        read_timeout: 10,
+      },
     }
 
     Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index c0d741d57..4cad93767 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -81,7 +81,9 @@ class ImportService < BaseService
       end
     end
 
-    Import::RelationshipWorker.push_bulk(items) do |acct, extra|
+    head_items = items.uniq { |acct, _| acct.split('@')[1] }
+    tail_items = items - head_items
+    Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
       [@account.id, acct, action, extra]
     end
   end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 17ace100c..ba77552c6 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -112,6 +112,8 @@ class ResolveAccountService < BaseService
   end
 
   def webfinger_update_due?
+    return false if @options[:check_delivery_availability] && !DeliveryFailureTracker.available?(@domain)
+
     @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
   end
 
diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb
index 616da6da9..4a455f3ae 100644
--- a/app/workers/import/relationship_worker.rb
+++ b/app/workers/import/relationship_worker.rb
@@ -7,7 +7,8 @@ class Import::RelationshipWorker
 
   def perform(account_id, target_account_uri, relationship, options = {})
     from_account   = Account.find(account_id)
-    target_account = ResolveAccountService.new.call(target_account_uri)
+    target_domain  = domain(target_account_uri)
+    target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) }
     options.symbolize_keys!
 
     return if target_account.nil?
@@ -29,4 +30,22 @@ class Import::RelationshipWorker
   rescue ActiveRecord::RecordNotFound
     true
   end
+
+  def domain(uri)
+    domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1]
+    TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain)
+  end
+
+  def stoplight_wrap_request(domain, &block)
+    if domain.present?
+      Stoplight("source:#{domain}", &block)
+        .with_fallback { nil }
+        .with_threshold(1)
+        .with_cool_off_time(5.minutes.seconds)
+        .with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) }
+        .run
+    else
+      block.call
+    end
+  end
 end
-- 
cgit 


From 89f40b6c3ec525b09d02f21e9b45276084167d8d Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 9 Jun 2020 10:32:00 +0200
Subject: Make domain block/silence/reject-media code more robust (#13424)

* Split media cleanup from reject-media domain blocks to its own service

* Slightly improve ClearDomainMediaService error handling

* Lower DomainClearMediaWorker to lowest-priority queue

* Do not catch ActiveRecord::RecordNotFound in domain block workers

* Fix DomainBlockWorker spec labels

* Add some specs

* Change domain blocks to immediately mark accounts as suspended

Rather than doing so sequentially, account after account, while cleaning
their data. This doesn't change much about the time the block takes to
complete, but it immediately prevents interaction with the blocked domain,
while up to now, it would only be guaranteed when the process ends.
---
 app/services/block_domain_service.rb             | 53 +-----------------
 app/services/clear_domain_media_service.rb       | 70 ++++++++++++++++++++++++
 app/workers/domain_block_worker.rb               |  7 ++-
 app/workers/domain_clear_media_worker.rb         | 14 +++++
 spec/services/clear_domain_media_service_spec.rb | 23 ++++++++
 spec/workers/domain_block_worker_spec.rb         |  4 +-
 spec/workers/domain_clear_media_worker_spec.rb   | 26 +++++++++
 7 files changed, 142 insertions(+), 55 deletions(-)
 create mode 100644 app/services/clear_domain_media_service.rb
 create mode 100644 app/workers/domain_clear_media_worker.rb
 create mode 100644 spec/services/clear_domain_media_service_spec.rb
 create mode 100644 spec/workers/domain_clear_media_worker_spec.rb

(limited to 'app/workers')

diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 9f0860674..dc23ef8d8 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -26,59 +26,20 @@ class BlockDomainService < BaseService
       suspend_accounts!
     end
 
-    clear_media! if domain_block.reject_media?
-  end
-
-  def invalidate_association_caches!
-    # Normally, associated models of a status are immutable (except for accounts)
-    # so they are aggressively cached. After updating the media attachments to no
-    # longer point to a local file, we need to clear the cache to make those
-    # changes appear in the API and UI
-    @affected_status_ids.each { |id| Rails.cache.delete_matched("statuses/#{id}-*") }
+    DomainClearMediaWorker.perform_async(domain_block.id) if domain_block.reject_media?
   end
 
   def silence_accounts!
     blocked_domain_accounts.without_silenced.in_batches.update_all(silenced_at: @domain_block.created_at)
   end
 
-  def clear_media!
-    @affected_status_ids = []
-
-    clear_account_images!
-    clear_account_attachments!
-    clear_emojos!
-
-    invalidate_association_caches!
-  end
-
   def suspend_accounts!
-    blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
+    blocked_domain_accounts.without_suspended.in_batches.update_all(suspended_at: @domain_block.created_at)
+    blocked_domain_accounts.where(suspended_at: @domain_block.created_at).reorder(nil).find_each do |account|
       SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at)
     end
   end
 
-  def clear_account_images!
-    blocked_domain_accounts.reorder(nil).find_each do |account|
-      account.avatar.destroy if account.avatar.exists?
-      account.header.destroy if account.header.exists?
-      account.save
-    end
-  end
-
-  def clear_account_attachments!
-    media_from_blocked_domain.reorder(nil).find_each do |attachment|
-      @affected_status_ids << attachment.status_id if attachment.status_id.present?
-
-      attachment.file.destroy if attachment.file.exists?
-      attachment.type = :unknown
-      attachment.save
-    end
-  end
-
-  def clear_emojos!
-    emojis_from_blocked_domains.destroy_all
-  end
-
   def blocked_domain
     domain_block.domain
   end
@@ -86,12 +47,4 @@ class BlockDomainService < BaseService
   def blocked_domain_accounts
     Account.by_domain_and_subdomains(blocked_domain)
   end
-
-  def media_from_blocked_domain
-    MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
-  end
-
-  def emojis_from_blocked_domains
-    CustomEmoji.by_domain_and_subdomains(blocked_domain)
-  end
 end
diff --git a/app/services/clear_domain_media_service.rb b/app/services/clear_domain_media_service.rb
new file mode 100644
index 000000000..704cfb71a
--- /dev/null
+++ b/app/services/clear_domain_media_service.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+class ClearDomainMediaService < BaseService
+  attr_reader :domain_block
+
+  def call(domain_block)
+    @domain_block = domain_block
+    clear_media! if domain_block.reject_media?
+  end
+
+  private
+
+  def invalidate_association_caches!
+    # Normally, associated models of a status are immutable (except for accounts)
+    # so they are aggressively cached. After updating the media attachments to no
+    # longer point to a local file, we need to clear the cache to make those
+    # changes appear in the API and UI
+    @affected_status_ids.each { |id| Rails.cache.delete_matched("statuses/#{id}-*") }
+  end
+
+  def clear_media!
+    @affected_status_ids = []
+
+    begin
+      clear_account_images!
+      clear_account_attachments!
+      clear_emojos!
+    ensure
+      invalidate_association_caches!
+    end
+  end
+
+  def clear_account_images!
+    blocked_domain_accounts.reorder(nil).find_each do |account|
+      account.avatar.destroy if account.avatar&.exists?
+      account.header.destroy if account.header&.exists?
+      account.save
+    end
+  end
+
+  def clear_account_attachments!
+    media_from_blocked_domain.reorder(nil).find_each do |attachment|
+      @affected_status_ids << attachment.status_id if attachment.status_id.present?
+
+      attachment.file.destroy if attachment.file&.exists?
+      attachment.type = :unknown
+      attachment.save
+    end
+  end
+
+  def clear_emojos!
+    emojis_from_blocked_domains.destroy_all
+  end
+
+  def blocked_domain
+    domain_block.domain
+  end
+
+  def blocked_domain_accounts
+    Account.by_domain_and_subdomains(blocked_domain)
+  end
+
+  def media_from_blocked_domain
+    MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
+  end
+
+  def emojis_from_blocked_domains
+    CustomEmoji.by_domain_and_subdomains(blocked_domain)
+  end
+end
diff --git a/app/workers/domain_block_worker.rb b/app/workers/domain_block_worker.rb
index 35518d6b5..3c601cd83 100644
--- a/app/workers/domain_block_worker.rb
+++ b/app/workers/domain_block_worker.rb
@@ -4,8 +4,9 @@ class DomainBlockWorker
   include Sidekiq::Worker
 
   def perform(domain_block_id, update = false)
-    BlockDomainService.new.call(DomainBlock.find(domain_block_id), update)
-  rescue ActiveRecord::RecordNotFound
-    true
+    domain_block = DomainBlock.find_by(id: domain_block_id)
+    return true if domain_block.nil?
+
+    BlockDomainService.new.call(domain_block, update)
   end
 end
diff --git a/app/workers/domain_clear_media_worker.rb b/app/workers/domain_clear_media_worker.rb
new file mode 100644
index 000000000..971934a08
--- /dev/null
+++ b/app/workers/domain_clear_media_worker.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class DomainClearMediaWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(domain_block_id)
+    domain_block = DomainBlock.find_by(id: domain_block_id)
+    return true if domain_block.nil?
+
+    ClearDomainMediaService.new.call(domain_block)
+  end
+end
diff --git a/spec/services/clear_domain_media_service_spec.rb b/spec/services/clear_domain_media_service_spec.rb
new file mode 100644
index 000000000..8e58c6039
--- /dev/null
+++ b/spec/services/clear_domain_media_service_spec.rb
@@ -0,0 +1,23 @@
+require 'rails_helper'
+
+RSpec.describe ClearDomainMediaService, type: :service do
+  let!(:bad_account) { Fabricate(:account, username: 'badguy666', domain: 'evil.org') }
+  let!(:bad_status1) { Fabricate(:status, account: bad_account, text: 'You suck') }
+  let!(:bad_status2) { Fabricate(:status, account: bad_account, text: 'Hahaha') }
+  let!(:bad_attachment) { Fabricate(:media_attachment, account: bad_account, status: bad_status2, file: attachment_fixture('attachment.jpg')) }
+
+  subject { ClearDomainMediaService.new }
+
+  describe 'for a silence with reject media' do
+    before do
+      subject.call(DomainBlock.create!(domain: 'evil.org', severity: :silence, reject_media: true))
+    end
+
+    it 'leaves the domains status and attachements, but clears media' do
+      expect { bad_status1.reload }.not_to raise_error
+      expect { bad_status2.reload }.not_to raise_error
+      expect { bad_attachment.reload }.not_to raise_error
+      expect(bad_attachment.file.exists?).to be false
+    end
+  end
+end
diff --git a/spec/workers/domain_block_worker_spec.rb b/spec/workers/domain_block_worker_spec.rb
index 48b3e38c4..bd8fc4a62 100644
--- a/spec/workers/domain_block_worker_spec.rb
+++ b/spec/workers/domain_block_worker_spec.rb
@@ -8,7 +8,7 @@ describe DomainBlockWorker do
   describe 'perform' do
     let(:domain_block) { Fabricate(:domain_block) }
 
-    it 'returns true for non-existent domain block' do
+    it 'calls domain block service for relevant domain block' do
       service = double(call: nil)
       allow(BlockDomainService).to receive(:new).and_return(service)
       result = subject.perform(domain_block.id)
@@ -17,7 +17,7 @@ describe DomainBlockWorker do
       expect(service).to have_received(:call).with(domain_block, false)
     end
 
-    it 'calls domain block service for relevant domain block' do
+    it 'returns true for non-existent domain block' do
       result = subject.perform('aaa')
 
       expect(result).to eq(true)
diff --git a/spec/workers/domain_clear_media_worker_spec.rb b/spec/workers/domain_clear_media_worker_spec.rb
new file mode 100644
index 000000000..36251b1ec
--- /dev/null
+++ b/spec/workers/domain_clear_media_worker_spec.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe DomainClearMediaWorker do
+  subject { described_class.new }
+
+  describe 'perform' do
+    let(:domain_block) { Fabricate(:domain_block, severity: :silence, reject_media: true) }
+
+    it 'calls domain clear media service for relevant domain block' do
+      service = double(call: nil)
+      allow(ClearDomainMediaService).to receive(:new).and_return(service)
+      result = subject.perform(domain_block.id)
+
+      expect(result).to be_nil
+      expect(service).to have_received(:call).with(domain_block)
+    end
+
+    it 'returns true for non-existent domain block' do
+      result = subject.perform('aaa')
+
+      expect(result).to eq(true)
+    end
+  end
+end
-- 
cgit