From 5c9abdeff1d0cf3e14d84c5ae298e6a5beccaf18 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 27 Sep 2022 03:08:19 +0200
Subject: Add retention policy for cached content and media (#19232)

---
 app/lib/vacuum/access_tokens_vacuum.rb     | 18 ++++++++++
 app/lib/vacuum/backups_vacuum.rb           | 25 ++++++++++++++
 app/lib/vacuum/feeds_vacuum.rb             | 34 +++++++++++++++++++
 app/lib/vacuum/media_attachments_vacuum.rb | 40 ++++++++++++++++++++++
 app/lib/vacuum/preview_cards_vacuum.rb     | 39 +++++++++++++++++++++
 app/lib/vacuum/statuses_vacuum.rb          | 54 ++++++++++++++++++++++++++++++
 app/lib/vacuum/system_keys_vacuum.rb       | 13 +++++++
 7 files changed, 223 insertions(+)
 create mode 100644 app/lib/vacuum/access_tokens_vacuum.rb
 create mode 100644 app/lib/vacuum/backups_vacuum.rb
 create mode 100644 app/lib/vacuum/feeds_vacuum.rb
 create mode 100644 app/lib/vacuum/media_attachments_vacuum.rb
 create mode 100644 app/lib/vacuum/preview_cards_vacuum.rb
 create mode 100644 app/lib/vacuum/statuses_vacuum.rb
 create mode 100644 app/lib/vacuum/system_keys_vacuum.rb

(limited to 'app/lib/vacuum')

diff --git a/app/lib/vacuum/access_tokens_vacuum.rb b/app/lib/vacuum/access_tokens_vacuum.rb
new file mode 100644
index 000000000..4f3878027
--- /dev/null
+++ b/app/lib/vacuum/access_tokens_vacuum.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Vacuum::AccessTokensVacuum
+  def perform
+    vacuum_revoked_access_tokens!
+    vacuum_revoked_access_grants!
+  end
+
+  private
+
+  def vacuum_revoked_access_tokens!
+    Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+  end
+
+  def vacuum_revoked_access_grants!
+    Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+  end
+end
diff --git a/app/lib/vacuum/backups_vacuum.rb b/app/lib/vacuum/backups_vacuum.rb
new file mode 100644
index 000000000..3b83072f3
--- /dev/null
+++ b/app/lib/vacuum/backups_vacuum.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class Vacuum::BackupsVacuum
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_expired_backups! if retention_period?
+  end
+
+  private
+
+  def vacuum_expired_backups!
+    backups_past_retention_period.in_batches.destroy_all
+  end
+
+  def backups_past_retention_period
+    Backup.unscoped.where(Backup.arel_table[:created_at].lt(@retention_period.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb
new file mode 100644
index 000000000..f46bcf75f
--- /dev/null
+++ b/app/lib/vacuum/feeds_vacuum.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class Vacuum::FeedsVacuum
+  def perform
+    vacuum_inactive_home_feeds!
+    vacuum_inactive_list_feeds!
+  end
+
+  private
+
+  def vacuum_inactive_home_feeds!
+    inactive_users.select(:id, :account_id).find_in_batches do |users|
+      feed_manager.clean_feeds!(:home, users.map(&:account_id))
+    end
+  end
+
+  def vacuum_inactive_list_feeds!
+    inactive_users_lists.select(:id).find_in_batches do |lists|
+      feed_manager.clean_feeds!(:list, lists.map(&:id))
+    end
+  end
+
+  def inactive_users
+    User.confirmed.inactive
+  end
+
+  def inactive_users_lists
+    List.where(account_id: inactive_users.select(:account_id))
+  end
+
+  def feed_manager
+    FeedManager.instance
+  end
+end
diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb
new file mode 100644
index 000000000..7fb347ce4
--- /dev/null
+++ b/app/lib/vacuum/media_attachments_vacuum.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Vacuum::MediaAttachmentsVacuum
+  TTL = 1.day.freeze
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_cached_files! if retention_period?
+    vacuum_orphaned_records!
+  end
+
+  private
+
+  def vacuum_cached_files!
+    media_attachments_past_retention_period.find_each do |media_attachment|
+      media_attachment.file.destroy
+      media_attachment.thumbnail.destroy
+      media_attachment.save
+    end
+  end
+
+  def vacuum_orphaned_records!
+    orphaned_media_attachments.in_batches.destroy_all
+  end
+
+  def media_attachments_past_retention_period
+    MediaAttachment.unscoped.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago))
+  end
+
+  def orphaned_media_attachments
+    MediaAttachment.unscoped.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb
new file mode 100644
index 000000000..84ef100ed
--- /dev/null
+++ b/app/lib/vacuum/preview_cards_vacuum.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Vacuum::PreviewCardsVacuum
+  TTL = 1.day.freeze
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_cached_images! if retention_period?
+    vacuum_orphaned_records!
+  end
+
+  private
+
+  def vacuum_cached_images!
+    preview_cards_past_retention_period.find_each do |preview_card|
+      preview_card.image.destroy
+      preview_card.save
+    end
+  end
+
+  def vacuum_orphaned_records!
+    orphaned_preview_cards.in_batches.destroy_all
+  end
+
+  def preview_cards_past_retention_period
+    PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago))
+  end
+
+  def orphaned_preview_cards
+    PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb
new file mode 100644
index 000000000..41d6ba270
--- /dev/null
+++ b/app/lib/vacuum/statuses_vacuum.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class Vacuum::StatusesVacuum
+  include Redisable
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_statuses! if retention_period?
+  end
+
+  private
+
+  def vacuum_statuses!
+    statuses_scope.find_in_batches do |statuses|
+      # Side-effects not covered by foreign keys, such
+      # as the search index, must be handled first.
+
+      remove_from_account_conversations(statuses)
+      remove_from_search_index(statuses)
+
+      # Foreign keys take care of most associated records
+      # for us. Media attachments will be orphaned.
+
+      Status.where(id: statuses.map(&:id)).delete_all
+    end
+  end
+
+  def statuses_scope
+    Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility)
+  end
+
+  def retention_period_as_id
+    Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false)
+  end
+
+  def analyze_statuses!
+    ActiveRecord::Base.connection.execute('ANALYZE statuses')
+  end
+
+  def remove_from_account_conversations(statuses)
+    Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations)
+  end
+
+  def remove_from_search_index(statuses)
+    with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled?
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/system_keys_vacuum.rb b/app/lib/vacuum/system_keys_vacuum.rb
new file mode 100644
index 000000000..ceee2fd16
--- /dev/null
+++ b/app/lib/vacuum/system_keys_vacuum.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Vacuum::SystemKeysVacuum
+  def perform
+    vacuum_expired_system_keys!
+  end
+
+  private
+
+  def vacuum_expired_system_keys!
+    SystemKey.expired.delete_all
+  end
+end
-- 
cgit 


From bf0ab3e0fac54515c13beef4ec09b0455f1bce67 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 26 Oct 2022 12:10:48 +0200
Subject: Fix vacuum scheduler missing lock, locks never expiring (#19458)

Remove vacuuming of orphaned preview cards
---
 app/lib/vacuum/media_attachments_vacuum.rb   | 2 +-
 app/lib/vacuum/preview_cards_vacuum.rb       | 9 ---------
 app/workers/scheduler/vacuum_scheduler.rb    | 2 +-
 config/initializers/sidekiq.rb               | 1 +
 spec/lib/vacuum/preview_cards_vacuum_spec.rb | 4 ----
 5 files changed, 3 insertions(+), 15 deletions(-)

(limited to 'app/lib/vacuum')

diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb
index 7fb347ce4..7c0a85a9d 100644
--- a/app/lib/vacuum/media_attachments_vacuum.rb
+++ b/app/lib/vacuum/media_attachments_vacuum.rb
@@ -8,8 +8,8 @@ class Vacuum::MediaAttachmentsVacuum
   end
 
   def perform
-    vacuum_cached_files! if retention_period?
     vacuum_orphaned_records!
+    vacuum_cached_files! if retention_period?
   end
 
   private
diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb
index 84ef100ed..14fdeda1c 100644
--- a/app/lib/vacuum/preview_cards_vacuum.rb
+++ b/app/lib/vacuum/preview_cards_vacuum.rb
@@ -9,7 +9,6 @@ class Vacuum::PreviewCardsVacuum
 
   def perform
     vacuum_cached_images! if retention_period?
-    vacuum_orphaned_records!
   end
 
   private
@@ -21,18 +20,10 @@ class Vacuum::PreviewCardsVacuum
     end
   end
 
-  def vacuum_orphaned_records!
-    orphaned_preview_cards.in_batches.destroy_all
-  end
-
   def preview_cards_past_retention_period
     PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago))
   end
 
-  def orphaned_preview_cards
-    PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago))
-  end
-
   def retention_period?
     @retention_period.present?
   end
diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb
index ce88ff204..9544f808b 100644
--- a/app/workers/scheduler/vacuum_scheduler.rb
+++ b/app/workers/scheduler/vacuum_scheduler.rb
@@ -3,7 +3,7 @@
 class Scheduler::VacuumScheduler
   include Sidekiq::Worker
 
-  sidekiq_options retry: 0
+  sidekiq_options retry: 0, lock: :until_executed
 
   def perform
     vacuum_operations.each do |operation|
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index c1327053d..9d2abf074 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -35,4 +35,5 @@ SidekiqUniqueJobs.configure do |config|
   config.reaper_count    = 1000
   config.reaper_interval = 600
   config.reaper_timeout  = 150
+  config.lock_ttl        = 50.days.to_i
 end
diff --git a/spec/lib/vacuum/preview_cards_vacuum_spec.rb b/spec/lib/vacuum/preview_cards_vacuum_spec.rb
index 4a4a599fa..275f9ba92 100644
--- a/spec/lib/vacuum/preview_cards_vacuum_spec.rb
+++ b/spec/lib/vacuum/preview_cards_vacuum_spec.rb
@@ -28,9 +28,5 @@ RSpec.describe Vacuum::PreviewCardsVacuum do
     it 'does not delete attached preview cards' do
       expect(new_preview_card.reload).to be_persisted
     end
-
-    it 'deletes preview cards not attached to any status' do
-      expect { orphaned_preview_card.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 end
-- 
cgit