From 78ef635980ff391ea3ba4c37de836947a97958a0 Mon Sep 17 00:00:00 2001 From: Evan <35814742+evanphilip@users.noreply.github.com> Date: Wed, 14 Dec 2022 19:50:07 +0100 Subject: Add command to remove avatar and header images of inactive remote accounts from the local database (#22149) * Add tootctl subcommand media remove-profile-media * Trigger workflows * Correcting external linting * External linting error * External linting fix * Merging with remove command * Linting * Correct long option names Co-authored-by: Claire * Correct long option names Co-authored-by: Claire * Correct long option names Co-authored-by: Claire * Remove saving a list of purged accounts Co-authored-by: Claire --- lib/mastodon/media_cli.rb | 78 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index bba4a1bd7..24cc98964 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -14,35 +14,78 @@ module Mastodon end option :days, type: :numeric, default: 7, aliases: [:d] + option :prune_profiles, type: :boolean, default: false + option :remove_headers, type: :boolean, default: false + option :include_follows, type: :boolean, default: false option :concurrency, type: :numeric, default: 5, aliases: [:c] - option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false - desc 'remove', 'Remove remote media files' + desc 'remove', 'Remove remote media files, headers or avatars' long_desc <<-DESC - Removes locally cached copies of media attachments from other servers. - + Removes locally cached copies of media attachments (and optionally profile + headers and avatars) from other servers. By default, only media attachements + are removed. The --days option specifies how old media attachments have to be before - they are removed. It defaults to 7 days. + they are removed. In case of avatars and headers, it specifies how old + the last webfinger request and update to the user has to be before they + are pruned. It defaults to 7 days. + If --prune-profiles is specified, only avatars and headers are removed. + If --remove-headers is specified, only headers are removed. + If --include-follows is specified along with --prune-profiles or + --remove-headers, all non-local profiles will be pruned irrespective of + follow status. By default, only accounts that are not followed by or + following anyone locally are pruned. DESC + # rubocop:disable Metrics/PerceivedComplexity def remove - time_ago = options[:days].days.ago - dry_run = options[:dry_run] ? '(DRY RUN)' : '' + if options[:prune_profiles] && options[:remove_headers] + say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true) + exit(1) + end + if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers]) + say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true) + exit(1) + end + time_ago = options[:days].days.ago + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' - processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| - next if media_attachment.file.blank? + if options[:prune_profiles] || options[:remove_headers] + processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account| + next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists? + next if account.avatar.blank? && account.header.blank? + next if options[:remove_headers] && account.header.blank? - size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) + size = (account.header_file_size || 0) + size += (account.avatar_file_size || 0) if options[:prune_profiles] - unless options[:dry_run] - media_attachment.file.destroy - media_attachment.thumbnail.destroy - media_attachment.save + unless options[:dry_run] + account.header.destroy + account.avatar.destroy if options[:prune_profiles] + account.save! + end + + size end - size + say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true) end - say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) + unless options[:prune_profiles] || options[:remove_headers] + processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment| + next if media_attachment.file.blank? + + size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) + + unless options[:dry_run] + media_attachment.file.destroy + media_attachment.thumbnail.destroy + media_attachment.save + end + + size + end + + say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) + end end option :start_after @@ -183,6 +226,7 @@ module Mastodon say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true) end + # rubocop:enable Metrics/PerceivedComplexity option :account, type: :string option :domain, type: :string @@ -269,7 +313,7 @@ module Mastodon def lookup(url) path = Addressable::URI.parse(url).path - path_segments = path.split('/')[2..-1] + path_segments = path.split('/')[2..] path_segments.delete('cache') unless [7, 10].include?(path_segments.size) -- cgit From f239d31f23b8bd55fb26f67906b815e4abe65d92 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 15 Dec 2022 14:52:50 +0100 Subject: Add --email and --dry-run options to `tootctl accounts delete` (#22328) --- lib/mastodon/accounts_cli.rb | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 77cbef84e..0dd852131 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -200,21 +200,44 @@ module Mastodon end end - desc 'delete USERNAME', 'Delete a user' + option :email + option :dry_run, type: :boolean + desc 'delete [USERNAME]', 'Delete a user' long_desc <<-LONG_DESC Remove a user account with a given USERNAME. - LONG_DESC - def delete(username) - account = Account.find_local(username) - if account.nil? - say('No user with such username', :red) + With the --email option, the user is selected based on email + rather than username. + LONG_DESC + def delete(username = nil) + if username.present? && options[:email].present? + say('Use username or --email, not both', :red) + exit(1) + elsif username.blank? && options[:email].blank? + say('No username provided', :red) exit(1) end - say("Deleting user with #{account.statuses_count} statuses, this might take a while...") - DeleteAccountService.new.call(account, reserve_email: false) - say('OK', :green) + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + account = nil + + if username.present? + account = Account.find_local(username) + if account.nil? + say('No user with such username', :red) + exit(1) + end + else + account = Account.left_joins(:user).find_by(user: { email: options[:email] }) + if account.nil? + say('No user with such email', :red) + exit(1) + end + end + + say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run}") + DeleteAccountService.new.call(account, reserve_email: false) unless options[:dry_run] + say("OK#{dry_run}", :green) end option :force, type: :boolean, aliases: [:f], description: 'Override public key check' -- cgit From 3d3429243fa0bacb20aac3db6c377441c0510f22 Mon Sep 17 00:00:00 2001 From: Dan Peterson Date: Thu, 15 Dec 2022 11:38:51 -0400 Subject: Fix default S3_HOSTNAME used in mastodon:setup (#19932) s3-us-east-1.amazonaws.com does not exist. Co-authored-by: Effy Elden --- lib/tasks/mastodon.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index c1e5bd2b4..3c891a07f 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -194,7 +194,7 @@ namespace :mastodon do env['S3_HOSTNAME'] = prompt.ask('S3 hostname:') do |q| q.required true - q.default 's3-us-east-1.amazonaws.com' + q.default 's3.us-east-1.amazonaws.com' q.modify :strip end -- cgit