From 91c929a42cbb71b47af976ec021f4be235b9f3fe Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 19 Aug 2018 15:48:29 +0200
Subject: Keep scheduler jobs unique until they're done (#8287)

---
 app/workers/scheduler/backup_cleanup_scheduler.rb        | 2 ++
 app/workers/scheduler/doorkeeper_cleanup_scheduler.rb    | 2 ++
 app/workers/scheduler/email_scheduler.rb                 | 2 ++
 app/workers/scheduler/feed_cleanup_scheduler.rb          | 2 ++
 app/workers/scheduler/ip_cleanup_scheduler.rb            | 2 ++
 app/workers/scheduler/media_cleanup_scheduler.rb         | 2 ++
 app/workers/scheduler/subscriptions_cleanup_scheduler.rb | 2 ++
 app/workers/scheduler/subscriptions_scheduler.rb         | 2 ++
 app/workers/scheduler/user_cleanup_scheduler.rb          | 2 ++
 9 files changed, 18 insertions(+)

(limited to 'app/workers')

diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb
index 5ab16c057..cfdf6f4af 100644
--- a/app/workers/scheduler/backup_cleanup_scheduler.rb
+++ b/app/workers/scheduler/backup_cleanup_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::BackupCleanupScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     old_backups.find_each(&:destroy!)
   end
diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
index bab4ae886..fec08c6bc 100644
--- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
+++ b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::DoorkeeperCleanupScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
     Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
diff --git a/app/workers/scheduler/email_scheduler.rb b/app/workers/scheduler/email_scheduler.rb
index 36866061b..1a5a1c826 100644
--- a/app/workers/scheduler/email_scheduler.rb
+++ b/app/workers/scheduler/email_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::EmailScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     eligible_users.find_each do |user|
       next unless user.allows_digest_emails?
diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb
index 42cf14128..b02bac883 100644
--- a/app/workers/scheduler/feed_cleanup_scheduler.rb
+++ b/app/workers/scheduler/feed_cleanup_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::FeedCleanupScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     clean_home_feeds!
     clean_list_feeds!
diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb
index 613a5e336..6bb93df7d 100644
--- a/app/workers/scheduler/ip_cleanup_scheduler.rb
+++ b/app/workers/scheduler/ip_cleanup_scheduler.rb
@@ -5,6 +5,8 @@ class Scheduler::IpCleanupScheduler
 
   RETENTION_PERIOD = 1.year
 
+  sidekiq_options unique: :until_executed
+
   def perform
     time_ago = RETENTION_PERIOD.ago
     SessionActivation.where('updated_at < ?', time_ago).destroy_all
diff --git a/app/workers/scheduler/media_cleanup_scheduler.rb b/app/workers/scheduler/media_cleanup_scheduler.rb
index c35686fcb..a27e02953 100644
--- a/app/workers/scheduler/media_cleanup_scheduler.rb
+++ b/app/workers/scheduler/media_cleanup_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::MediaCleanupScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     unattached_media.find_each(&:destroy)
   end
diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
index af2ae3120..06ba66205 100644
--- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::SubscriptionsCleanupScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     Subscription.expired.in_batches.delete_all
   end
diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb
index dc16e85c2..4b0959af2 100644
--- a/app/workers/scheduler/subscriptions_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::SubscriptionsScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id))
   end
diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb
index 245536cea..dde195bff 100644
--- a/app/workers/scheduler/user_cleanup_scheduler.rb
+++ b/app/workers/scheduler/user_cleanup_scheduler.rb
@@ -3,6 +3,8 @@
 class Scheduler::UserCleanupScheduler
   include Sidekiq::Worker
 
+  sidekiq_options unique: :until_executed
+
   def perform
     User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).find_in_batches do |batch|
       Account.where(id: batch.map(&:account_id)).delete_all
-- 
cgit 


From d98de8ada743886c3cd48b2ad942d46b805af7a9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 21 Aug 2018 12:25:50 +0200
Subject: Get rid of all batch order warnings (#8334)

---
 app/models/form/status_batch.rb                         | 4 ++--
 app/services/after_block_domain_from_account_service.rb | 4 ++--
 app/services/backup_service.rb                          | 4 ++--
 app/services/block_domain_service.rb                    | 6 +++---
 app/services/remove_status_service.rb                   | 4 ++--
 app/workers/refollow_worker.rb                          | 2 +-
 app/workers/scheduler/backup_cleanup_scheduler.rb       | 2 +-
 app/workers/scheduler/email_scheduler.rb                | 2 +-
 app/workers/scheduler/user_cleanup_scheduler.rb         | 2 +-
 lib/tasks/mastodon.rake                                 | 8 ++++----
 10 files changed, 19 insertions(+), 19 deletions(-)

(limited to 'app/workers')

diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index 4f08a3049..8f5fd1fa2 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -23,7 +23,7 @@ class Form::StatusBatch
     media_attached_status_ids = MediaAttachment.where(status_id: status_ids).pluck(:status_id)
 
     ApplicationRecord.transaction do
-      Status.where(id: media_attached_status_ids).find_each do |status|
+      Status.where(id: media_attached_status_ids).reorder(nil).find_each do |status|
         status.update!(sensitive: sensitive)
         log_action :update, status
       end
@@ -35,7 +35,7 @@ class Form::StatusBatch
   end
 
   def delete_statuses
-    Status.where(id: status_ids).find_each do |status|
+    Status.where(id: status_ids).reorder(nil).find_each do |status|
       RemovalWorker.perform_async(status.id)
       log_action :destroy, status
     end
diff --git a/app/services/after_block_domain_from_account_service.rb b/app/services/after_block_domain_from_account_service.rb
index 0f1a8505d..56cc819fb 100644
--- a/app/services/after_block_domain_from_account_service.rb
+++ b/app/services/after_block_domain_from_account_service.rb
@@ -15,13 +15,13 @@ class AfterBlockDomainFromAccountService < BaseService
   private
 
   def reject_existing_followers!
-    @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).find_each do |follow|
+    @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow|
       reject_follow!(follow)
     end
   end
 
   def reject_pending_follow_requests!
-    FollowRequest.where(target_account: @account).where(account: Account.where(domain: @domain)).includes(:account).find_each do |follow_request|
+    FollowRequest.where(target_account: @account).where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow_request|
       reject_follow!(follow_request)
     end
   end
diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 8492c1117..da7db6462 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -18,7 +18,7 @@ class BackupService < BaseService
   def build_json!
     @collection = serialize(collection_presenter, ActivityPub::CollectionSerializer)
 
-    account.statuses.with_includes.find_in_batches do |statuses|
+    account.statuses.with_includes.reorder(nil).find_in_batches do |statuses|
       statuses.each do |status|
         item = serialize(status, ActivityPub::ActivitySerializer)
         item.delete(:'@context')
@@ -60,7 +60,7 @@ class BackupService < BaseService
   end
 
   def dump_media_attachments!(tar)
-    MediaAttachment.attached.where(account: account).find_in_batches do |media_attachments|
+    MediaAttachment.attached.where(account: account).reorder(nil).find_in_batches do |media_attachments|
       media_attachments.each do |m|
         download_to_tar(tar, m.file, m.file.path)
       end
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index d082de40b..a1fe93665 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -43,14 +43,14 @@ class BlockDomainService < BaseService
   end
 
   def suspend_accounts!
-    blocked_domain_accounts.where(suspended: false).find_each do |account|
+    blocked_domain_accounts.where(suspended: false).reorder(nil).find_each do |account|
       UnsubscribeService.new.call(account) if account.subscribed?
       SuspendAccountService.new.call(account)
     end
   end
 
   def clear_account_images!
-    blocked_domain_accounts.find_each do |account|
+    blocked_domain_accounts.reorder(nil).find_each do |account|
       account.avatar.destroy if account.avatar.exists?
       account.header.destroy if account.header.exists?
       account.save
@@ -58,7 +58,7 @@ class BlockDomainService < BaseService
   end
 
   def clear_account_attachments!
-    media_from_blocked_domain.find_each do |attachment|
+    media_from_blocked_domain.reorder(nil).find_each do |attachment|
       @affected_status_ids << attachment.status_id if attachment.status_id.present?
 
       attachment.file.destroy if attachment.file.exists?
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index fb889140b..1a53093b8 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -43,13 +43,13 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_followers
-    @account.followers_for_local_distribution.find_each do |follower|
+    @account.followers_for_local_distribution.reorder(nil).find_each do |follower|
       FeedManager.instance.unpush_from_home(follower, @status)
     end
   end
 
   def remove_from_lists
-    @account.lists_for_local_distribution.select(:id, :account_id).find_each do |list|
+    @account.lists_for_local_distribution.select(:id, :account_id).reorder(nil).find_each do |list|
       FeedManager.instance.unpush_from_list(list, @status)
     end
   end
diff --git a/app/workers/refollow_worker.rb b/app/workers/refollow_worker.rb
index 66bcd27c3..12f2bf671 100644
--- a/app/workers/refollow_worker.rb
+++ b/app/workers/refollow_worker.rb
@@ -9,7 +9,7 @@ class RefollowWorker
     target_account = Account.find(target_account_id)
     return unless target_account.protocol == :activitypub
 
-    target_account.followers.where(domain: nil).find_each do |follower|
+    target_account.followers.where(domain: nil).reorder(nil).find_each do |follower|
       # Locally unfollow remote account
       follower.unfollow!(target_account)
 
diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb
index cfdf6f4af..023a77307 100644
--- a/app/workers/scheduler/backup_cleanup_scheduler.rb
+++ b/app/workers/scheduler/backup_cleanup_scheduler.rb
@@ -6,7 +6,7 @@ class Scheduler::BackupCleanupScheduler
   sidekiq_options unique: :until_executed
 
   def perform
-    old_backups.find_each(&:destroy!)
+    old_backups.reorder(nil).find_each(&:destroy!)
   end
 
   private
diff --git a/app/workers/scheduler/email_scheduler.rb b/app/workers/scheduler/email_scheduler.rb
index 1a5a1c826..24117e424 100644
--- a/app/workers/scheduler/email_scheduler.rb
+++ b/app/workers/scheduler/email_scheduler.rb
@@ -6,7 +6,7 @@ class Scheduler::EmailScheduler
   sidekiq_options unique: :until_executed
 
   def perform
-    eligible_users.find_each do |user|
+    eligible_users.reorder(nil).find_each do |user|
       next unless user.allows_digest_emails?
       DigestMailerWorker.perform_async(user.id)
     end
diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb
index dde195bff..626fb1652 100644
--- a/app/workers/scheduler/user_cleanup_scheduler.rb
+++ b/app/workers/scheduler/user_cleanup_scheduler.rb
@@ -6,7 +6,7 @@ class Scheduler::UserCleanupScheduler
   sidekiq_options unique: :until_executed
 
   def perform
-    User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).find_in_batches do |batch|
+    User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).reorder(nil).find_in_batches do |batch|
       Account.where(id: batch.map(&:account_id)).delete_all
       User.where(id: batch.map(&:id)).delete_all
     end
diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index f693c8b5a..191ce634c 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -503,7 +503,7 @@ namespace :mastodon do
     desc 'Remove media attachments attributed to silenced accounts'
     task remove_silenced: :environment do
       nb_media_attachments = 0
-      MediaAttachment.where(account: Account.silenced).select(:id).find_in_batches do |media_attachments|
+      MediaAttachment.where(account: Account.silenced).select(:id).reorder(nil).find_in_batches do |media_attachments|
         nb_media_attachments += media_attachments.length
         Maintenance::DestroyMediaWorker.push_bulk(media_attachments.map(&:id))
       end
@@ -515,7 +515,7 @@ namespace :mastodon do
       time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
       nb_media_attachments = 0
 
-      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).find_in_batches do |media_attachments|
+      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
         nb_media_attachments += media_attachments.length
         Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
       end
@@ -535,7 +535,7 @@ namespace :mastodon do
       accounts = accounts.where(domain: ENV['DOMAIN']) if ENV['DOMAIN'].present?
       nb_accounts = 0
 
-      accounts.select(:id).find_in_batches do |accounts_batch|
+      accounts.select(:id).reorder(nil).find_in_batches do |accounts_batch|
         nb_accounts += accounts_batch.length
         Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts_batch.map(&:id))
       end
@@ -570,7 +570,7 @@ namespace :mastodon do
 
     desc 'Generates home timelines for users who logged in in the past two weeks'
     task build: :environment do
-      User.active.select(:id, :account_id).find_in_batches do |users|
+      User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users|
         RegenerationWorker.push_bulk(users.map(&:account_id))
       end
     end
-- 
cgit 


From f06fa099625e928e5858ea81a20be1eddf6c6fbb Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 21 Aug 2018 17:53:01 +0200
Subject: Revert to using Paperclip's filesystem storage, and fix dangling
 records in remove_remote (#8339)

* Fix uncaching worker

* Revert to using Paperclip's filesystem backend instead of fog-local

fog-local has lots of concurrency issues, causing failure to delete files,
dangling file records, and spurious errors UncacheMediaWorker
---
 Gemfile                                         |  1 -
 Gemfile.lock                                    |  3 ---
 app/workers/maintenance/uncache_media_worker.rb |  2 +-
 config/initializers/paperclip.rb                | 12 ++++--------
 4 files changed, 5 insertions(+), 13 deletions(-)

(limited to 'app/workers')

diff --git a/Gemfile b/Gemfile
index 31c3c8086..516d397a2 100644
--- a/Gemfile
+++ b/Gemfile
@@ -16,7 +16,6 @@ gem 'dotenv-rails', '~> 2.2', '< 2.3'
 
 gem 'aws-sdk-s3', '~> 1.9', require: false
 gem 'fog-core', '~> 1.45'
-gem 'fog-local', '~> 0.5', require: false
 gem 'fog-openstack', '~> 0.1', require: false
 gem 'paperclip', '~> 6.0'
 gem 'paperclip-av-transcoder', '~> 0.6'
diff --git a/Gemfile.lock b/Gemfile.lock
index fbffc0c2d..ea0e3f0cd 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -220,8 +220,6 @@ GEM
     fog-json (1.0.2)
       fog-core (~> 1.0)
       multi_json (~> 1.10)
-    fog-local (0.5.0)
-      fog-core (>= 1.27, < 3.0)
     fog-openstack (0.1.25)
       fog-core (~> 1.40)
       fog-json (>= 1.0)
@@ -679,7 +677,6 @@ DEPENDENCIES
   fast_blank (~> 1.0)
   fastimage
   fog-core (~> 1.45)
-  fog-local (~> 0.5)
   fog-openstack (~> 0.1)
   fuubar (~> 2.2)
   goldfinger (~> 2.1)
diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb
index f6a51a1b8..2d1a670a7 100644
--- a/app/workers/maintenance/uncache_media_worker.rb
+++ b/app/workers/maintenance/uncache_media_worker.rb
@@ -8,7 +8,7 @@ class Maintenance::UncacheMediaWorker
   def perform(media_attachment_id)
     media = MediaAttachment.find(media_attachment_id)
 
-    return unless media.file.exists?
+    return if media.file.blank?
 
     media.file.destroy
     media.save
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
index c134bc5b8..59ab9b9a1 100644
--- a/config/initializers/paperclip.rb
+++ b/config/initializers/paperclip.rb
@@ -74,14 +74,10 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
     fog_public: true
   )
 else
-  require 'fog/local'
-
   Paperclip::Attachment.default_options.merge!(
-    fog_credentials: {
-      provider: 'Local',
-      local_root: ENV.fetch('PAPERCLIP_ROOT_PATH') { Rails.root.join('public', 'system') },
-    },
-    fog_directory: '',
-    fog_host: ENV.fetch('PAPERCLIP_ROOT_URL') { '/system' }
+    storage: :filesystem,
+    use_timestamp: true,
+    path: (ENV['PAPERCLIP_ROOT_PATH'] || ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    url: (ENV['PAPERCLIP_ROOT_URL'] || '/system') + '/:class/:attachment/:id_partition/:style/:filename',
   )
 end
-- 
cgit