diff options
author | ThibG <thib@sitedethib.com> | 2019-12-12 16:10:58 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-12-12 16:10:58 +0100 |
commit | c1befd3909359e4d964da74bde9931a8416d26a3 (patch) | |
tree | 7eb7528245c671cdf908e07c3d630e7639d575bb /lib | |
parent | 246addd5b33a172600342af3fb6fb5e4c80ad95e (diff) | |
parent | be4849c08306cce930ea27e1f8097ffb774f75c6 (diff) |
Merge pull request #1252 from ThibG/glitch-soc/merge-upstream
Merge upstream changes
Diffstat (limited to 'lib')
-rw-r--r-- | lib/mastodon/media_cli.rb | 99 |
1 files changed, 99 insertions, 0 deletions
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 3b702f155..d842b986f 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -44,6 +44,105 @@ module Mastodon say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) end + option :start_after + option :dry_run, type: :boolean, default: false + desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments' + long_desc <<~LONG_DESC + Scans file storage for files that do not belong to existing media attachments. Because this operation + requires iterating over every single file individually, it will be slow. + + Please mind that some storage providers charge for the necessary API requests to list objects. + LONG_DESC + def remove_orphans + progress = create_progress_bar(nil) + reclaimed_bytes = 0 + removed = 0 + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + + case Paperclip::Attachment.default_options[:storage] + when :s3 + paperclip_instance = MediaAttachment.new.file + s3_interface = paperclip_instance.s3_interface + bucket = s3_interface.bucket(Paperclip::Attachment.default_options[:s3_credentials][:bucket]) + last_key = options[:start_after] + + loop do + objects = begin + begin + bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x } + rescue => e + progress.log(pastel.red("Error fetching list of files: #{e}")) + progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key + break + end + end + + break if objects.empty? + + last_key = objects.last.key + attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment } + + objects.each do |object| + attachment_id = object.key.split('/')[2..-2].join.to_i + filename = object.key.split('/').last + + progress.increment + + next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename) + + begin + object.delete unless options[:dry_run] + + reclaimed_bytes += object.size + removed += 1 + + progress.log("Found and removed orphan: #{object.key}") + rescue => e + progress.log(pastel.red("Error processing #{object.key}: #{e}")) + end + end + end + when :fog + say('The fog storage driver is not supported for this operation at this time', :red) + exit(1) + when :filesystem + require 'find' + + root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s) + + Find.find(File.join(root_path, 'media_attachments', 'files')) do |path| + next if File.directory?(path) + + key = path.gsub("#{root_path}#{File::SEPARATOR}", '') + attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i + filename = key.split(File::SEPARATOR).last + attachment = MediaAttachment.find_by(id: attachment_id) + + progress.increment + + next unless attachment.nil? || !attachment.variant?(filename) + + begin + size = File.size(path) + + File.delete(path) unless options[:dry_run] + + reclaimed_bytes += size + removed += 1 + + progress.log("Found and removed orphan: #{key}") + rescue => e + progress.log(pastel.red("Error processing #{key}: #{e}")) + end + end + end + + progress.total = progress.progress + progress.finish + + say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true) + end + option :account, type: :string option :domain, type: :string option :status, type: :numeric |