From 5c9abdeff1d0cf3e14d84c5ae298e6a5beccaf18 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 27 Sep 2022 03:08:19 +0200 Subject: Add retention policy for cached content and media (#19232) --- spec/lib/vacuum/access_tokens_vacuum_spec.rb | 33 +++++++++++++++++ spec/lib/vacuum/backups_vacuum_spec.rb | 24 ++++++++++++ spec/lib/vacuum/feeds_vacuum_spec.rb | 30 +++++++++++++++ spec/lib/vacuum/media_attachments_vacuum_spec.rb | 47 ++++++++++++++++++++++++ spec/lib/vacuum/preview_cards_vacuum_spec.rb | 36 ++++++++++++++++++ spec/lib/vacuum/statuses_vacuum_spec.rb | 36 ++++++++++++++++++ spec/lib/vacuum/system_keys_vacuum_spec.rb | 22 +++++++++++ 7 files changed, 228 insertions(+) create mode 100644 spec/lib/vacuum/access_tokens_vacuum_spec.rb create mode 100644 spec/lib/vacuum/backups_vacuum_spec.rb create mode 100644 spec/lib/vacuum/feeds_vacuum_spec.rb create mode 100644 spec/lib/vacuum/media_attachments_vacuum_spec.rb create mode 100644 spec/lib/vacuum/preview_cards_vacuum_spec.rb create mode 100644 spec/lib/vacuum/statuses_vacuum_spec.rb create mode 100644 spec/lib/vacuum/system_keys_vacuum_spec.rb (limited to 'spec/lib/vacuum') diff --git a/spec/lib/vacuum/access_tokens_vacuum_spec.rb b/spec/lib/vacuum/access_tokens_vacuum_spec.rb new file mode 100644 index 000000000..0244c3449 --- /dev/null +++ b/spec/lib/vacuum/access_tokens_vacuum_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe Vacuum::AccessTokensVacuum do + subject { described_class.new } + + describe '#perform' do + let!(:revoked_access_token) { Fabricate(:access_token, revoked_at: 1.minute.ago) } + let!(:active_access_token) { Fabricate(:access_token) } + + let!(:revoked_access_grant) { Fabricate(:access_grant, revoked_at: 1.minute.ago) } + let!(:active_access_grant) { Fabricate(:access_grant) } + + before do + subject.perform + end + + it 'deletes revoked access tokens' do + expect { revoked_access_token.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'deletes revoked access grants' do + expect { revoked_access_grant.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete active access tokens' do + expect { active_access_token.reload }.to_not raise_error + end + + it 'does not delete active access grants' do + expect { active_access_grant.reload }.to_not raise_error + end + end +end diff --git a/spec/lib/vacuum/backups_vacuum_spec.rb b/spec/lib/vacuum/backups_vacuum_spec.rb new file mode 100644 index 000000000..4e2de083f --- /dev/null +++ b/spec/lib/vacuum/backups_vacuum_spec.rb @@ -0,0 +1,24 @@ +require 'rails_helper' + +RSpec.describe Vacuum::BackupsVacuum do + let(:retention_period) { 7.days } + + subject { described_class.new(retention_period) } + + describe '#perform' do + let!(:expired_backup) { Fabricate(:backup, created_at: (retention_period + 1.day).ago) } + let!(:current_backup) { Fabricate(:backup) } + + before do + subject.perform + end + + it 'deletes backups past the retention period' do + expect { expired_backup.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete backups within the retention period' do + expect { current_backup.reload }.to_not raise_error + end + end +end diff --git a/spec/lib/vacuum/feeds_vacuum_spec.rb b/spec/lib/vacuum/feeds_vacuum_spec.rb new file mode 100644 index 000000000..0aec26740 --- /dev/null +++ b/spec/lib/vacuum/feeds_vacuum_spec.rb @@ -0,0 +1,30 @@ +require 'rails_helper' + +RSpec.describe Vacuum::FeedsVacuum do + subject { described_class.new } + + describe '#perform' do + let!(:active_user) { Fabricate(:user, current_sign_in_at: 2.days.ago) } + let!(:inactive_user) { Fabricate(:user, current_sign_in_at: 22.days.ago) } + + before do + redis.zadd(feed_key_for(inactive_user), 1, 1) + redis.zadd(feed_key_for(active_user), 1, 1) + redis.zadd(feed_key_for(inactive_user, 'reblogs'), 2, 2) + redis.sadd(feed_key_for(inactive_user, 'reblogs:2'), 3) + + subject.perform + end + + it 'clears feeds of inactive users and lists' do + expect(redis.zcard(feed_key_for(inactive_user))).to eq 0 + expect(redis.zcard(feed_key_for(active_user))).to eq 1 + expect(redis.exists?(feed_key_for(inactive_user, 'reblogs'))).to be false + expect(redis.exists?(feed_key_for(inactive_user, 'reblogs:2'))).to be false + end + end + + def feed_key_for(user, subtype = nil) + FeedManager.instance.key(:home, user.account_id, subtype) + end +end diff --git a/spec/lib/vacuum/media_attachments_vacuum_spec.rb b/spec/lib/vacuum/media_attachments_vacuum_spec.rb new file mode 100644 index 000000000..be8458d9b --- /dev/null +++ b/spec/lib/vacuum/media_attachments_vacuum_spec.rb @@ -0,0 +1,47 @@ +require 'rails_helper' + +RSpec.describe Vacuum::MediaAttachmentsVacuum do + let(:retention_period) { 7.days } + + subject { described_class.new(retention_period) } + + let(:remote_status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com')) } + let(:local_status) { Fabricate(:status) } + + describe '#perform' do + let!(:old_remote_media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', status: remote_status, created_at: (retention_period + 1.day).ago, updated_at: (retention_period + 1.day).ago) } + let!(:old_local_media) { Fabricate(:media_attachment, status: local_status, created_at: (retention_period + 1.day).ago, updated_at: (retention_period + 1.day).ago) } + let!(:new_remote_media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', status: remote_status) } + let!(:new_local_media) { Fabricate(:media_attachment, status: local_status) } + let!(:old_unattached_media) { Fabricate(:media_attachment, account_id: nil, created_at: 10.days.ago) } + let!(:new_unattached_media) { Fabricate(:media_attachment, account_id: nil, created_at: 1.hour.ago) } + + before do + subject.perform + end + + it 'deletes cache of remote media attachments past the retention period' do + expect(old_remote_media.reload.file).to be_blank + end + + it 'does not touch local media attachments past the retention period' do + expect(old_local_media.reload.file).to_not be_blank + end + + it 'does not delete cache of remote media attachments within the retention period' do + expect(new_remote_media.reload.file).to_not be_blank + end + + it 'does not touch local media attachments within the retention period' do + expect(new_local_media.reload.file).to_not be_blank + end + + it 'deletes unattached media attachments past TTL' do + expect { old_unattached_media.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'does not delete unattached media attachments within TTL' do + expect(new_unattached_media.reload).to be_persisted + end + end +end diff --git a/spec/lib/vacuum/preview_cards_vacuum_spec.rb b/spec/lib/vacuum/preview_cards_vacuum_spec.rb new file mode 100644 index 000000000..4a4a599fa --- /dev/null +++ b/spec/lib/vacuum/preview_cards_vacuum_spec.rb @@ -0,0 +1,36 @@ +require 'rails_helper' + +RSpec.describe Vacuum::PreviewCardsVacuum do + let(:retention_period) { 7.days } + + subject { described_class.new(retention_period) } + + describe '#perform' do + let!(:orphaned_preview_card) { Fabricate(:preview_card, created_at: 2.days.ago) } + let!(:old_preview_card) { Fabricate(:preview_card, updated_at: (retention_period + 1.day).ago) } + let!(:new_preview_card) { Fabricate(:preview_card) } + + before do + old_preview_card.statuses << Fabricate(:status) + new_preview_card.statuses << Fabricate(:status) + + subject.perform + end + + it 'deletes cache of preview cards last updated before the retention period' do + expect(old_preview_card.reload.image).to be_blank + end + + it 'does not delete cache of preview cards last updated within the retention period' do + expect(new_preview_card.reload.image).to_not be_blank + end + + it 'does not delete attached preview cards' do + expect(new_preview_card.reload).to be_persisted + end + + it 'deletes preview cards not attached to any status' do + expect { orphaned_preview_card.reload }.to raise_error ActiveRecord::RecordNotFound + end + end +end diff --git a/spec/lib/vacuum/statuses_vacuum_spec.rb b/spec/lib/vacuum/statuses_vacuum_spec.rb new file mode 100644 index 000000000..83f3c5c9f --- /dev/null +++ b/spec/lib/vacuum/statuses_vacuum_spec.rb @@ -0,0 +1,36 @@ +require 'rails_helper' + +RSpec.describe Vacuum::StatusesVacuum do + let(:retention_period) { 7.days } + + let(:remote_account) { Fabricate(:account, domain: 'example.com') } + + subject { described_class.new(retention_period) } + + describe '#perform' do + let!(:remote_status_old) { Fabricate(:status, account: remote_account, created_at: (retention_period + 2.days).ago) } + let!(:remote_status_recent) { Fabricate(:status, account: remote_account, created_at: (retention_period - 2.days).ago) } + let!(:local_status_old) { Fabricate(:status, created_at: (retention_period + 2.days).ago) } + let!(:local_status_recent) { Fabricate(:status, created_at: (retention_period - 2.days).ago) } + + before do + subject.perform + end + + it 'deletes remote statuses past the retention period' do + expect { remote_status_old.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete local statuses past the retention period' do + expect { local_status_old.reload }.to_not raise_error + end + + it 'does not delete remote statuses within the retention period' do + expect { remote_status_recent.reload }.to_not raise_error + end + + it 'does not delete local statuses within the retention period' do + expect { local_status_recent.reload }.to_not raise_error + end + end +end diff --git a/spec/lib/vacuum/system_keys_vacuum_spec.rb b/spec/lib/vacuum/system_keys_vacuum_spec.rb new file mode 100644 index 000000000..565892f02 --- /dev/null +++ b/spec/lib/vacuum/system_keys_vacuum_spec.rb @@ -0,0 +1,22 @@ +require 'rails_helper' + +RSpec.describe Vacuum::SystemKeysVacuum do + subject { described_class.new } + + describe '#perform' do + let!(:expired_system_key) { Fabricate(:system_key, created_at: (SystemKey::ROTATION_PERIOD * 4).ago) } + let!(:current_system_key) { Fabricate(:system_key) } + + before do + subject.perform + end + + it 'deletes the expired key' do + expect { expired_system_key.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete the current key' do + expect { current_system_key.reload }.to_not raise_error + end + end +end -- cgit From bf0ab3e0fac54515c13beef4ec09b0455f1bce67 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 26 Oct 2022 12:10:48 +0200 Subject: Fix vacuum scheduler missing lock, locks never expiring (#19458) Remove vacuuming of orphaned preview cards --- app/lib/vacuum/media_attachments_vacuum.rb | 2 +- app/lib/vacuum/preview_cards_vacuum.rb | 9 --------- app/workers/scheduler/vacuum_scheduler.rb | 2 +- config/initializers/sidekiq.rb | 1 + spec/lib/vacuum/preview_cards_vacuum_spec.rb | 4 ---- 5 files changed, 3 insertions(+), 15 deletions(-) (limited to 'spec/lib/vacuum') diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb index 7fb347ce4..7c0a85a9d 100644 --- a/app/lib/vacuum/media_attachments_vacuum.rb +++ b/app/lib/vacuum/media_attachments_vacuum.rb @@ -8,8 +8,8 @@ class Vacuum::MediaAttachmentsVacuum end def perform - vacuum_cached_files! if retention_period? vacuum_orphaned_records! + vacuum_cached_files! if retention_period? end private diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb index 84ef100ed..14fdeda1c 100644 --- a/app/lib/vacuum/preview_cards_vacuum.rb +++ b/app/lib/vacuum/preview_cards_vacuum.rb @@ -9,7 +9,6 @@ class Vacuum::PreviewCardsVacuum def perform vacuum_cached_images! if retention_period? - vacuum_orphaned_records! end private @@ -21,18 +20,10 @@ class Vacuum::PreviewCardsVacuum end end - def vacuum_orphaned_records! - orphaned_preview_cards.in_batches.destroy_all - end - def preview_cards_past_retention_period PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago)) end - def orphaned_preview_cards - PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago)) - end - def retention_period? @retention_period.present? end diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb index ce88ff204..9544f808b 100644 --- a/app/workers/scheduler/vacuum_scheduler.rb +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::VacuumScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed def perform vacuum_operations.each do |operation| diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index c1327053d..9d2abf074 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -35,4 +35,5 @@ SidekiqUniqueJobs.configure do |config| config.reaper_count = 1000 config.reaper_interval = 600 config.reaper_timeout = 150 + config.lock_ttl = 50.days.to_i end diff --git a/spec/lib/vacuum/preview_cards_vacuum_spec.rb b/spec/lib/vacuum/preview_cards_vacuum_spec.rb index 4a4a599fa..275f9ba92 100644 --- a/spec/lib/vacuum/preview_cards_vacuum_spec.rb +++ b/spec/lib/vacuum/preview_cards_vacuum_spec.rb @@ -28,9 +28,5 @@ RSpec.describe Vacuum::PreviewCardsVacuum do it 'does not delete attached preview cards' do expect(new_preview_card.reload).to be_persisted end - - it 'deletes preview cards not attached to any status' do - expect { orphaned_preview_card.reload }.to raise_error ActiveRecord::RecordNotFound - end end end -- cgit