From 4ac78e2a066508a54de82f1d910ef2fd36c3d106 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 9 Aug 2021 23:11:50 +0200 Subject: Add feature to automatically delete old toots (#16529) * Add account statuses cleanup policy model * Record last inspected toot to delete to speed up successive calls to statuses_to_delete * Add service to cleanup a given account's statuses within a budget * Add worker to go through account policies and delete old toots * Fix last inspected status id logic All existing statuses older or equal to last inspected status id must be kept by the current policy. This is an invariant that must be kept so that resuming deletion from the last inspected status remains sound. * Add tests * Refactor scheduler and add tests * Add user interface * Add support for discriminating based on boosts/favs * Add UI support for min_reblogs and min_favs, rework UI * Address first round of review comments * Replace Snowflake#id_at_start with with_random parameter * Add tests * Add tests for StatusesCleanupController * Rework settings page * Adjust load-avoiding mechanisms * Please CodeClimate --- .../accounts_statuses_cleanup_scheduler_spec.rb | 127 +++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb (limited to 'spec/workers/scheduler') diff --git a/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb new file mode 100644 index 000000000..8f20725c8 --- /dev/null +++ b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb @@ -0,0 +1,127 @@ +require 'rails_helper' + +describe Scheduler::AccountsStatusesCleanupScheduler do + subject { described_class.new } + + let!(:account1) { Fabricate(:account, domain: nil) } + let!(:account2) { Fabricate(:account, domain: nil) } + let!(:account3) { Fabricate(:account, domain: nil) } + let!(:account4) { Fabricate(:account, domain: nil) } + let!(:remote) { Fabricate(:account) } + + let!(:policy1) { Fabricate(:account_statuses_cleanup_policy, account: account1) } + let!(:policy2) { Fabricate(:account_statuses_cleanup_policy, account: account3) } + let!(:policy3) { Fabricate(:account_statuses_cleanup_policy, account: account4, enabled: false) } + + let(:queue_size) { 0 } + let(:queue_latency) { 0 } + let(:process_set_stub) do + [ + { + 'concurrency' => 2, + 'queues' => ['push', 'default'], + }, + ] + end + let(:retry_size) { 0 } + + before do + queue_stub = double + allow(queue_stub).to receive(:size).and_return(queue_size) + allow(queue_stub).to receive(:latency).and_return(queue_latency) + allow(Sidekiq::Queue).to receive(:new).and_return(queue_stub) + allow(Sidekiq::ProcessSet).to receive(:new).and_return(process_set_stub) + + sidekiq_stats_stub = double + allow(sidekiq_stats_stub).to receive(:retry_size).and_return(retry_size) + allow(Sidekiq::Stats).to receive(:new).and_return(sidekiq_stats_stub) + + # Create a bunch of old statuses + 10.times do + Fabricate(:status, account: account1, created_at: 3.years.ago) + Fabricate(:status, account: account2, created_at: 3.years.ago) + Fabricate(:status, account: account3, created_at: 3.years.ago) + Fabricate(:status, account: account4, created_at: 3.years.ago) + Fabricate(:status, account: remote, created_at: 3.years.ago) + end + + # Create a bunch of newer statuses + 5.times do + Fabricate(:status, account: account1, created_at: 3.minutes.ago) + Fabricate(:status, account: account2, created_at: 3.minutes.ago) + Fabricate(:status, account: account3, created_at: 3.minutes.ago) + Fabricate(:status, account: account4, created_at: 3.minutes.ago) + Fabricate(:status, account: remote, created_at: 3.minutes.ago) + end + end + + describe '#under_load?' do + context 'when nothing is queued' do + it 'returns false' do + expect(subject.under_load?).to be false + end + end + + context 'when numerous jobs are queued' do + let(:queue_size) { 5 } + let(:queue_latency) { 120 } + + it 'returns true' do + expect(subject.under_load?).to be true + end + end + + context 'when there is a huge amount of jobs to retry' do + let(:retry_size) { 1_000_000 } + + it 'returns true' do + expect(subject.under_load?).to be true + end + end + end + + describe '#get_budget' do + context 'on a single thread' do + let(:process_set_stub) { [ { 'concurrency' => 1, 'queues' => ['push', 'default'] } ] } + + it 'returns a low value' do + expect(subject.compute_budget).to be < 10 + end + end + + context 'on a lot of threads' do + let(:process_set_stub) do + [ + { 'concurrency' => 2, 'queues' => ['push', 'default'] }, + { 'concurrency' => 2, 'queues' => ['push'] }, + { 'concurrency' => 2, 'queues' => ['push'] }, + { 'concurrency' => 2, 'queues' => ['push'] }, + ] + end + + it 'returns a larger value' do + expect(subject.compute_budget).to be > 10 + end + end + end + + describe '#perform' do + context 'when the budget is lower than the number of toots to delete' do + it 'deletes as many statuses as the given budget' do + expect { subject.perform }.to change { Status.count }.by(-subject.compute_budget) + end + + it 'does not delete from accounts with no cleanup policy' do + expect { subject.perform }.to_not change { account2.statuses.count } + end + + it 'does not delete from accounts with disabled cleanup policies' do + expect { subject.perform }.to_not change { account4.statuses.count } + end + + it 'eventually deletes every deletable toot' do + expect { subject.perform; subject.perform; subject.perform; subject.perform }.to change { Status.count }.by(-20) + end + end + end +end -- cgit