From cb5b5cb5f79bb2187d8124df91af4c8e1bfd7256 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 2 May 2018 18:58:48 +0200
Subject: Slightly reduce RAM usage (#7301)

* No need to re-require sidekiq plugins, they are required via Gemfile

* Add derailed_benchmarks tool, no need to require TTY gems in Gemfile

* Replace ruby-oembed with FetchOEmbedService

Reduce startup by 45382 allocated objects

* Remove preloaded JSON-LD in favour of caching HTTP responses

Reduce boot RAM by about 6 MiB

* Fix tests

* Fix test suite by stubbing out JSON-LD contexts
---
 app/workers/scheduler/backup_cleanup_scheduler.rb        | 1 -
 app/workers/scheduler/doorkeeper_cleanup_scheduler.rb    | 1 -
 app/workers/scheduler/email_scheduler.rb                 | 1 -
 app/workers/scheduler/feed_cleanup_scheduler.rb          | 1 -
 app/workers/scheduler/ip_cleanup_scheduler.rb            | 1 -
 app/workers/scheduler/media_cleanup_scheduler.rb         | 1 -
 app/workers/scheduler/subscriptions_cleanup_scheduler.rb | 2 --
 app/workers/scheduler/subscriptions_scheduler.rb         | 3 ---
 app/workers/scheduler/user_cleanup_scheduler.rb          | 1 -
 app/workers/soft_block_domain_followers_worker.rb        | 2 --
 10 files changed, 14 deletions(-)

(limited to 'app/workers')

diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb
index 7a9d4f894..5ab16c057 100644
--- a/app/workers/scheduler/backup_cleanup_scheduler.rb
+++ b/app/workers/scheduler/backup_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::BackupCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
index 6488798cd..bab4ae886 100644
--- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
+++ b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::DoorkeeperCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/email_scheduler.rb b/app/workers/scheduler/email_scheduler.rb
index 24d0c0ebe..36866061b 100644
--- a/app/workers/scheduler/email_scheduler.rb
+++ b/app/workers/scheduler/email_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::EmailScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb
index 23fa7672b..42cf14128 100644
--- a/app/workers/scheduler/feed_cleanup_scheduler.rb
+++ b/app/workers/scheduler/feed_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::FeedCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb
index a33ca031e..613a5e336 100644
--- a/app/workers/scheduler/ip_cleanup_scheduler.rb
+++ b/app/workers/scheduler/ip_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::IpCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/media_cleanup_scheduler.rb b/app/workers/scheduler/media_cleanup_scheduler.rb
index ce32ce314..c35686fcb 100644
--- a/app/workers/scheduler/media_cleanup_scheduler.rb
+++ b/app/workers/scheduler/media_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::MediaCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
index 3b9211e81..af2ae3120 100644
--- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-scheduler'
-
 class Scheduler::SubscriptionsCleanupScheduler
   include Sidekiq::Worker
 
diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb
index 469a3d2a6..dc16e85c2 100644
--- a/app/workers/scheduler/subscriptions_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_scheduler.rb
@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-scheduler'
-require 'sidekiq-bulk'
-
 class Scheduler::SubscriptionsScheduler
   include Sidekiq::Worker
 
diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb
index a8f8fbd83..245536cea 100644
--- a/app/workers/scheduler/user_cleanup_scheduler.rb
+++ b/app/workers/scheduler/user_cleanup_scheduler.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'sidekiq-scheduler'
 
 class Scheduler::UserCleanupScheduler
   include Sidekiq::Worker
diff --git a/app/workers/soft_block_domain_followers_worker.rb b/app/workers/soft_block_domain_followers_worker.rb
index ce76683c5..85445c7fb 100644
--- a/app/workers/soft_block_domain_followers_worker.rb
+++ b/app/workers/soft_block_domain_followers_worker.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class SoftBlockDomainFollowersWorker
   include Sidekiq::Worker
 
-- 
cgit 


From 658cbc94255a91453fbadd175cd45ad15efcbdf3 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 2 May 2018 22:10:57 +0200
Subject: Improve PostStatusService performance (#7317)

Offload creation of local notifications to a worker. Remove two
redundant SQL queries from ProcessMentionsService, remove n+1
XML/JSON serialization via memoization
---
 app/services/process_mentions_service.rb | 44 ++++++++++++++++++--------------
 app/workers/local_notification_worker.rb | 12 +++++++++
 2 files changed, 37 insertions(+), 19 deletions(-)
 create mode 100644 app/workers/local_notification_worker.rb

(limited to 'app/workers')

diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index dc8df4a9a..2ed6698cf 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -10,55 +10,61 @@ class ProcessMentionsService < BaseService
   def call(status)
     return unless status.local?
 
+    @status  = status
+    mentions = []
+
     status.text = status.text.gsub(Account::MENTION_RE) do |match|
-      username, domain  = $1.split('@')
+      username, domain  = Regexp.last_match(1).split('@')
       mentioned_account = Account.find_remote(username, domain)
 
-      if mention_undeliverable?(status, mentioned_account)
+      if mention_undeliverable?(mentioned_account)
         begin
-          mentioned_account = resolve_account_service.call($1)
+          mentioned_account = resolve_account_service.call(Regexp.last_match(1))
         rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError
           mentioned_account = nil
         end
       end
 
-      next match if mention_undeliverable?(status, mentioned_account)
+      next match if mention_undeliverable?(mentioned_account)
+
+      mentions << mentioned_account.mentions.where(status: status).first_or_create(status: status)
 
-      mentioned_account.mentions.where(status: status).first_or_create(status: status)
       "@#{mentioned_account.acct}"
     end
 
     status.save!
 
-    status.mentions.includes(:account).each do |mention|
-      create_notification(status, mention)
-    end
+    mentions.each { |mention| create_notification(mention) }
   end
 
   private
 
-  def mention_undeliverable?(status, mentioned_account)
-    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus? && status.stream_entry.hidden?)
+  def mention_undeliverable?(mentioned_account)
+    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus? && @status.stream_entry.hidden?)
   end
 
-  def create_notification(status, mention)
+  def create_notification(mention)
     mentioned_account = mention.account
 
     if mentioned_account.local?
-      NotifyService.new.call(mentioned_account, mention)
-    elsif mentioned_account.ostatus? && !status.stream_entry.hidden?
-      NotificationWorker.perform_async(stream_entry_to_xml(status.stream_entry), status.account_id, mentioned_account.id)
+      LocalNotificationWorker.perform_async(mention.id)
+    elsif mentioned_account.ostatus? && !@status.stream_entry.hidden?
+      NotificationWorker.perform_async(ostatus_xml, @status.account_id, mentioned_account.id)
     elsif mentioned_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(mention.status), mention.status.account_id, mentioned_account.inbox_url)
+      ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url)
     end
   end
 
-  def build_json(status)
-    Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
-      status,
+  def ostatus_xml
+    @ostatus_xml ||= stream_entry_to_xml(@status.stream_entry)
+  end
+
+  def activitypub_json
+    @activitypub_json ||= Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
+      @status,
       serializer: ActivityPub::ActivitySerializer,
       adapter: ActivityPub::Adapter
-    ).as_json).sign!(status.account))
+    ).as_json).sign!(@status.account))
   end
 
   def resolve_account_service
diff --git a/app/workers/local_notification_worker.rb b/app/workers/local_notification_worker.rb
new file mode 100644
index 000000000..748270563
--- /dev/null
+++ b/app/workers/local_notification_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class LocalNotificationWorker
+  include Sidekiq::Worker
+
+  def perform(mention_id)
+    mention = Mention.find(mention_id)
+    NotifyService.new.call(mention.account, mention)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
-- 
cgit 


From bd367918328daedb37f49727f4e16e33679fdb15 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 3 May 2018 11:42:52 +0200
Subject: Do not override timestamps for incoming toots (#7326)

---
 app/workers/activitypub/processing_worker.rb | 2 +-
 app/workers/processing_worker.rb             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/workers')

diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb
index 0e2e0eddd..bb9adf64b 100644
--- a/app/workers/activitypub/processing_worker.rb
+++ b/app/workers/activitypub/processing_worker.rb
@@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true)
+    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id))
   end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 978c3aba2..5df404bcc 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -6,6 +6,6 @@ class ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
+    ProcessFeedService.new.call(body, Account.find(account_id))
   end
 end
-- 
cgit 


From d5fa4fbcd2649341f3f2809060c5a64e25ebfa81 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 3 May 2018 12:32:33 +0200
Subject: Revert "Do not override timestamps for incoming toots" (#7330)

* Revert "Weblate translations 20180503 (#7325)"

This reverts commit dfa6bccb64d9ee5512dddc10afd9a484db2dbb25.

* Revert "Prevent timeline from moving when cursor is hovering over it (fixes #7278) (#7327)"

This reverts commit 58852695c8ec490239ed3812f82971f8c1e6c172.

* Revert "Add pry-byebug (#7307)"

This reverts commit ab773e4d5ffdd78a61d3ebf0f79e60ee5c9f7e92.

* Revert "Do not override timestamps for incoming toots (#7326)"

This reverts commit bd367918328daedb37f49727f4e16e33679fdb15.
---
 app/workers/activitypub/processing_worker.rb | 2 +-
 app/workers/processing_worker.rb             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/workers')

diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb
index bb9adf64b..0e2e0eddd 100644
--- a/app/workers/activitypub/processing_worker.rb
+++ b/app/workers/activitypub/processing_worker.rb
@@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id))
+    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc..978c3aba2 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -6,6 +6,6 @@ class ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id))
+    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end
-- 
cgit 


From 581a5c9d29ef2a12f46b67a1097a9ad6df1c6953 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 3 May 2018 13:33:08 +0200
Subject: Fixes/do not override timestamps (#7331)

* Do not override timestamps for incoming toots

* Remove every reference to override_timestamps

Statuses are now created with the announced publishing date
and are only pushed to timelines if that date is at most
6 hours earlier than the time at which it is processed.
---
 app/lib/activitypub/activity.rb              | 2 +-
 app/lib/activitypub/activity/announce.rb     | 2 +-
 app/lib/activitypub/activity/create.rb       | 2 +-
 app/lib/ostatus/activity/creation.rb         | 4 ++--
 app/workers/activitypub/processing_worker.rb | 2 +-
 app/workers/processing_worker.rb             | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'app/workers')

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 9b00f0f52..5b97a6208 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -80,7 +80,7 @@ class ActivityPub::Activity
 
     # Only continue if the status is supposed to have
     # arrived in real-time
-    return unless @options[:override_timestamps] || status.within_realtime_window?
+    return unless status.within_realtime_window?
 
     distribute_to_followers(status)
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index c8a358195..8840a450c 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -15,7 +15,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       account: @account,
       reblog: original_status,
       uri: @json['id'],
-      created_at: @options[:override_timestamps] ? nil : @json['published'],
+      created_at: @json['published'],
       visibility: original_status.visibility
     )
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 411286fa5..edee2691f 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -47,7 +47,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       text: text_from_content || '',
       language: detected_language,
       spoiler_text: @object['summary'] || '',
-      created_at: @options[:override_timestamps] ? nil : @object['published'],
+      created_at: @object['published'],
       reply: @object['inReplyTo'].present?,
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 6235127b2..a24a0093c 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -39,7 +39,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
         reblog: cached_reblog,
         text: content,
         spoiler_text: content_warning,
-        created_at: @options[:override_timestamps] ? nil : published,
+        created_at: published,
         reply: thread?,
         language: content_language,
         visibility: visibility_scope,
@@ -61,7 +61,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
 
     LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-    DistributionWorker.perform_async(status.id) if @options[:override_timestamps] || status.within_realtime_window?
+    DistributionWorker.perform_async(status.id) if status.within_realtime_window?
 
     status
   end
diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb
index 0e2e0eddd..bb9adf64b 100644
--- a/app/workers/activitypub/processing_worker.rb
+++ b/app/workers/activitypub/processing_worker.rb
@@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true)
+    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id))
   end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 978c3aba2..5df404bcc 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -6,6 +6,6 @@ class ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
+    ProcessFeedService.new.call(body, Account.find(account_id))
   end
 end
-- 
cgit