Merge branch 'master' into patch-1

15 files changed, 90 insertions, 55 deletions

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 88f6f4a46..58d9fb1fc 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -34,12 +34,7 @@ class FeedManager
       trim(timeline_type, account.id)
     end
 
-    broadcast(account.id, event: 'update', payload: inline_render(account, 'api/v1/statuses/show', status))
-  end
-
-  def broadcast(timeline_id, options = {})
-    options[:queued_at] = (Time.now.to_f * 1000.0).to_i
-    redis.publish("timeline:#{timeline_id}", Oj.dump(options))
+    PushUpdateWorker.perform_async(account.id, status.id)
   end
 
   def trim(type, account_id)
@@ -81,10 +76,6 @@ class FeedManager
     end
   end
 
-  def inline_render(target_account, template, object)
-    Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: InlineRablScope.new(target_account)).render
-  end
-
   private
 
   def redis
diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb
new file mode 100644
index 000000000..8e04ad1d5
--- /dev/null
+++ b/app/lib/inline_renderer.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class InlineRenderer
+  def self.render(status, current_account, template)
+    Rabl::Renderer.new(
+      template,
+      status,
+      view_path: 'app/views',
+      format: :json,
+      scope: InlineRablScope.new(current_account)
+    ).render
+  end
+end
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 106d257ba..19eedc0a7 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -50,22 +50,23 @@ class FanOutOnWriteService < BaseService
   end
 
   def render_anonymous_payload(status)
-    @payload = FeedManager.instance.inline_render(nil, 'api/v1/statuses/show', status)
+    @payload = InlineRenderer.render(status, nil, 'api/v1/statuses/show')
+    @payload = Oj.dump(event: :update, payload: @payload)
   end
 
   def deliver_to_hashtags(status)
     Rails.logger.debug "Delivering status #{status.id} to hashtags"
 
     status.tags.pluck(:name).each do |hashtag|
-      FeedManager.instance.broadcast("hashtag:#{hashtag}", event: 'update', payload: @payload)
-      FeedManager.instance.broadcast("hashtag:#{hashtag}:local", event: 'update', payload: @payload) if status.account.local?
+      Redis.current.publish("timeline:hashtag:#{hashtag}", @payload)
+      Redis.current.publish("timeline:hashtag:#{hashtag}:local", @payload) if status.local?
     end
   end
 
   def deliver_to_public(status)
     Rails.logger.debug "Delivering status #{status.id} to public timeline"
 
-    FeedManager.instance.broadcast(:public, event: 'update', payload: @payload)
-    FeedManager.instance.broadcast('public:local', event: 'update', payload: @payload) if status.account.local?
+    Redis.current.publish('timeline:public', @payload)
+    Redis.current.publish('timeline:public:local', @payload) if status.local?
   end
 end
diff --git a/app/services/follow_remote_account_service.rb b/app/services/follow_remote_account_service.rb
index b39eafc70..936953429 100644
--- a/app/services/follow_remote_account_service.rb
+++ b/app/services/follow_remote_account_service.rb
@@ -45,13 +45,13 @@ class FollowRemoteAccountService < BaseService
     account.suspended   = true if domain_block && domain_block.suspend?
     account.silenced    = true if domain_block && domain_block.silence?
 
-    xml  = get_feed(account.remote_url)
-    hubs = get_hubs(xml)
+    body, xml = get_feed(account.remote_url)
+    hubs      = get_hubs(xml)
 
     account.uri     = get_account_uri(xml)
     account.hub_url = hubs.first.attribute('href').value
 
-    get_profile(xml, account)
+    get_profile(body, account)
     account.save!
 
     account
@@ -61,7 +61,7 @@ class FollowRemoteAccountService < BaseService
 
   def get_feed(url)
     response = http_client.get(Addressable::URI.parse(url))
-    Nokogiri::XML(response)
+    [response.to_s, Nokogiri::XML(response)]
   end
 
   def get_hubs(xml)
@@ -82,12 +82,8 @@ class FollowRemoteAccountService < BaseService
     author_uri.content
   end
 
-  def get_profile(xml, account)
-    update_remote_profile_service.call(xml.at_xpath('/xmlns:feed'), account)
-  end
-
-  def update_remote_profile_service
-    @update_remote_profile_service ||= UpdateRemoteProfileService.new
+  def get_profile(body, account)
+    RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), false)
   end
 
   def http_client
diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index 24486f220..ffeee5fcf 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -50,7 +50,7 @@ class NotifyService < BaseService
   def create_notification
     @notification.save!
     return unless @notification.browserable?
-    FeedManager.instance.broadcast(@recipient.id, event: 'notification', payload: FeedManager.instance.inline_render(@recipient, 'api/v1/notifications/show', @notification))
+    Redis.current.publish("timeline:#{@recipient.id}", Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, 'api/v1/notifications/show')))
   end
 
   def send_email
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
index 69911abc5..cf2f7a826 100644
--- a/app/services/process_feed_service.rb
+++ b/app/services/process_feed_service.rb
@@ -5,15 +5,15 @@ class ProcessFeedService < BaseService
     xml = Nokogiri::XML(body)
     xml.encoding = 'utf-8'
 
-    update_author(xml, account)
+    update_author(body, xml, account)
     process_entries(xml, account)
   end
 
   private
 
-  def update_author(xml, account)
+  def update_author(body, xml, account)
     return if xml.at_xpath('/xmlns:feed', xmlns: TagManager::XMLNS).nil?
-    UpdateRemoteProfileService.new.call(xml.at_xpath('/xmlns:feed', xmlns: TagManager::XMLNS), account, true)
+    RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
   end
 
   def process_entries(xml, account)
diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb
index d5f7b4b3c..805ca5a27 100644
--- a/app/services/process_interaction_service.rb
+++ b/app/services/process_interaction_service.rb
@@ -24,7 +24,7 @@ class ProcessInteractionService < BaseService
     return if account.suspended?
 
     if salmon.verify(envelope, account.keypair)
-      update_remote_profile_service.call(xml.at_xpath('/xmlns:entry', xmlns: TagManager::XMLNS), account, true)
+      RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
 
       case verb(xml)
       when :follow
@@ -114,7 +114,7 @@ class ProcessInteractionService < BaseService
 
     return if status.nil?
 
-    remove_status_service.call(status) if account.id == status.account_id
+    RemovalWorker.perform_async(status.id) if account.id == status.account_id
   end
 
   def favourite!(xml, from_account)
@@ -130,7 +130,7 @@ class ProcessInteractionService < BaseService
   end
 
   def add_post!(body, account)
-    process_feed_service.call(body, account)
+    ProcessingWorker.perform_async(account.id, body.force_encoding('UTF-8'))
   end
 
   def status(xml)
@@ -153,10 +153,6 @@ class ProcessInteractionService < BaseService
     @process_feed_service ||= ProcessFeedService.new
   end
 
-  def update_remote_profile_service
-    @update_remote_profile_service ||= UpdateRemoteProfileService.new
-  end
-
   def remove_status_service
     @remove_status_service ||= RemoveStatusService.new
   end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index cf1f432e4..50bb7fc97 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -4,6 +4,8 @@ class RemoveStatusService < BaseService
   include StreamEntryRenderer
 
   def call(status)
+    @payload = Oj.dump(event: :delete, payload: status.id)
+
     remove_from_self(status) if status.account.local?
     remove_from_followers(status)
     remove_from_mentioned(status)
@@ -25,25 +27,23 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_followers(status)
-    status.account.followers.each do |follower|
-      next unless follower.local?
+    status.account.followers.where(domain: nil).each do |follower|
       unpush(:home, follower, status)
     end
   end
 
   def remove_from_mentioned(status)
+    return unless status.local?
     notified_domains = []
 
     status.mentions.each do |mention|
       mentioned_account = mention.account
 
-      if mentioned_account.local?
-        unpush(:mentions, mentioned_account, status)
-      else
-        next if notified_domains.include?(mentioned_account.domain)
-        notified_domains << mentioned_account.domain
-        send_delete_salmon(mentioned_account, status)
-      end
+      next if mentioned_account.local?
+      next if notified_domains.include?(mentioned_account.domain)
+
+      notified_domains << mentioned_account.domain
+      send_delete_salmon(mentioned_account, status)
     end
   end
 
@@ -65,17 +65,19 @@ class RemoveStatusService < BaseService
       redis.zremrangebyscore(FeedManager.instance.key(type, receiver.id), status.id, status.id)
     end
 
-    FeedManager.instance.broadcast(receiver.id, event: 'delete', payload: status.id)
+    Redis.current.publish("timeline:#{receiver.id}", @payload)
   end
 
   def remove_from_hashtags(status)
-    status.tags.each do |tag|
-      FeedManager.instance.broadcast("hashtag:#{tag.name}", event: 'delete', payload: status.id)
+    status.tags.pluck(:name) do |hashtag|
+      Redis.current.publish("timeline:hashtag:#{hashtag}", @payload)
+      Redis.current.publish("timeline:hashtag:#{hashtag}:local", @payload) if status.local?
     end
   end
 
   def remove_from_public(status)
-    FeedManager.instance.broadcast(:public, event: 'delete', payload: status.id)
+    Redis.current.publish('timeline:public', @payload)
+    Redis.current.publish('timeline:public:local', @payload) if status.local?
   end
 
   def redis
diff --git a/app/workers/admin/suspension_worker.rb b/app/workers/admin/suspension_worker.rb
index 38761f3b9..7ef2b35ec 100644
--- a/app/workers/admin/suspension_worker.rb
+++ b/app/workers/admin/suspension_worker.rb
@@ -3,6 +3,8 @@
 class Admin::SuspensionWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull'
+
   def perform(account_id)
     SuspendAccountService.new.call(Account.find(account_id))
   end
diff --git a/app/workers/application_worker.rb b/app/workers/application_worker.rb
index f2d7c1062..436f24763 100644
--- a/app/workers/application_worker.rb
+++ b/app/workers/application_worker.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ApplicationWorker
   def info(message)
     Rails.logger.info("#{self.class.name} - #{message}")
diff --git a/app/workers/distribution_worker.rb b/app/workers/distribution_worker.rb
index 9a2867ea6..f7953689b 100644
--- a/app/workers/distribution_worker.rb
+++ b/app/workers/distribution_worker.rb
@@ -4,10 +4,7 @@ class DistributionWorker < ApplicationWorker
   include Sidekiq::Worker
 
   def perform(status_id)
-    status = Status.find(status_id)
-
-    FanOutOnWriteService.new.call(status)
-    WarmCacheService.new.call(status)
+    FanOutOnWriteService.new.call(Status.find(status_id))
   rescue ActiveRecord::RecordNotFound
     info("Couldn't find the status")
   end
diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb
index 7cf29fb53..d5a33cada 100644
--- a/app/workers/import_worker.rb
+++ b/app/workers/import_worker.rb
@@ -46,7 +46,7 @@ class ImportWorker
 
       begin
         FollowService.new.call(from_account, row[0])
-      rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError
+      rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound, Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError
         next
       end
     end
diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb
new file mode 100644
index 000000000..fbcdcf634
--- /dev/null
+++ b/app/workers/push_update_worker.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class PushUpdateWorker
+  include Sidekiq::Worker
+
+  def perform(account_id, status_id)
+    account = Account.find(account_id)
+    status  = Status.find(status_id)
+    message = InlineRenderer.render(status, account, 'api/v1/statuses/show')
+
+    Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/app/workers/remote_profile_update_worker.rb b/app/workers/remote_profile_update_worker.rb
new file mode 100644
index 000000000..b91dc3466
--- /dev/null
+++ b/app/workers/remote_profile_update_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class RemoteProfileUpdateWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(account_id, body, resubscribe)
+    account = Account.find(account_id)
+
+    xml = Nokogiri::XML(body)
+    xml.encoding = 'utf-8'
+
+    author_container = xml.at_xpath('/xmlns:feed', xmlns: TagManager::XMLNS) || xml.at_xpath('/xmlns:entry', xmlns: TagManager::XMLNS)
+
+    UpdateRemoteProfileService.new.call(author_container, account, resubscribe)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index fc95ce47f..d37d40432 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -7,7 +7,7 @@ class SalmonWorker
 
   def perform(account_id, body)
     ProcessInteractionService.new.call(body, Account.find(account_id))
-  rescue ActiveRecord::RecordNotFound
+  rescue Nokogiri::XML::XPath::SyntaxError, ActiveRecord::RecordNotFound
     true
   end
 end