Merge branch 'glitch'

6 files changed, 164 insertions, 37 deletions

diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
index d842b986f..08a8f1093 100644
--- a/lib/mastodon/media_cli.rb
+++ b/lib/mastodon/media_cli.rb
@@ -45,6 +45,7 @@ module Mastodon
     end
 
     option :start_after
+    option :prefix
     option :dry_run, type: :boolean, default: false
     desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
     long_desc <<~LONG_DESC
@@ -58,6 +59,7 @@ module Mastodon
       reclaimed_bytes = 0
       removed         = 0
       dry_run         = options[:dry_run] ? ' (DRY RUN)' : ''
+      prefix          = options[:prefix]
 
       case Paperclip::Attachment.default_options[:storage]
       when :s3
@@ -69,7 +71,7 @@ module Mastodon
         loop do
           objects = begin
             begin
-              bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x }
+              bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
             rescue => e
               progress.log(pastel.red("Error fetching list of files: #{e}"))
               progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
@@ -79,16 +81,21 @@ module Mastodon
 
           break if objects.empty?
 
-          last_key        = objects.last.key
-          attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment }
+          last_key   = objects.last.key
+          record_map = preload_records_from_mixed_objects(objects)
 
           objects.each do |object|
-            attachment_id = object.key.split('/')[2..-2].join.to_i
-            filename      = object.key.split('/').last
+            path_segments   = object.key.split('/')
+            model_name      = path_segments.first.classify
+            attachment_name = path_segments[1].singularize
+            record_id       = path_segments[2..-2].join.to_i
+            file_name       = path_segments.last
+            record          = record_map.dig(model_name, record_id)
+            attachment      = record&.public_send(attachment_name)
 
             progress.increment
 
-            next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename)
+            next unless attachment.blank? || !attachment.variant?(file_name)
 
             begin
               object.delete unless options[:dry_run]
@@ -108,19 +115,26 @@ module Mastodon
       when :filesystem
         require 'find'
 
-        root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
+        root_path = ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
 
-        Find.find(File.join(root_path, 'media_attachments', 'files')) do |path|
+        Find.find(File.join(*[root_path, prefix].compact)) do |path|
           next if File.directory?(path)
 
-          key           = path.gsub("#{root_path}#{File::SEPARATOR}", '')
-          attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i
-          filename      = key.split(File::SEPARATOR).last
-          attachment    = MediaAttachment.find_by(id: attachment_id)
+          key             = path.gsub("#{root_path}#{File::SEPARATOR}", '')
+          path_segments   = key.split(File::SEPARATOR)
+          model_name      = path_segments.first.classify
+          record_id       = path_segments[2..-2].join.to_i
+          attachment_name = path_segments[1].singularize
+          file_name       = path_segments.last
+
+          next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
+
+          record     = model_name.constantize.find_by(id: record_id)
+          attachment = record&.public_send(attachment_name)
 
           progress.increment
 
-          next unless attachment.nil? || !attachment.variant?(filename)
+          next unless attachment.blank? || !attachment.variant?(file_name)
 
           begin
             size = File.size(path)
@@ -213,25 +227,66 @@ module Mastodon
       say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
     end
 
-    desc 'lookup', 'Lookup where media is displayed by passing a media URL'
-    def lookup
-      prompt = TTY::Prompt.new
+    desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
+    def lookup(url)
+      path          = Addressable::URI.parse(url).path
+      path_segments = path.split('/')[2..-1]
+      model_name    = path_segments.first.classify
+      record_id     = path_segments[2..-2].join.to_i
 
-      url = prompt.ask('Please enter a URL to the media to lookup:', required: true)
+      unless PRELOAD_MODEL_WHITELIST.include?(model_name)
+        say("Cannot find corresponding model: #{model_name}", :red)
+        exit(1)
+      end
 
-      attachment_id = url
-                      .split('/')[0..-2]
-                      .grep(/\A\d+\z/)
-                      .join('')
+      record = model_name.constantize.find_by(id: record_id)
+      record = record.status if record.respond_to?(:status)
 
-      if url.split('/')[0..-2].include? 'media_attachments'
-        model = MediaAttachment.find(attachment_id).status
-        prompt.say(ActivityPub::TagManager.instance.url_for(model))
-      elsif url.split('/')[0..-2].include? 'accounts'
-        model = Account.find(attachment_id)
-        prompt.say(ActivityPub::TagManager.instance.url_for(model))
-      else
-        prompt.say('Not found')
+      unless record
+        say('Cannot find corresponding record', :red)
+        exit(1)
+      end
+
+      display_url = ActivityPub::TagManager.instance.url_for(record)
+
+      if display_url.blank?
+        say('No public URL for this type of record', :red)
+        exit(1)
+      end
+
+      say(display_url, :blue)
+    rescue Addressable::URI::InvalidURIError
+      say('Invalid URL', :red)
+      exit(1)
+    end
+
+    private
+
+    PRELOAD_MODEL_WHITELIST = %w(
+      Account
+      Backup
+      CustomEmoji
+      Import
+      MediaAttachment
+      PreviewCard
+      SiteUpload
+    ).freeze
+
+    def preload_records_from_mixed_objects(objects)
+      preload_map = Hash.new { |hash, key| hash[key] = [] }
+
+      objects.map do |object|
+        segments   = object.key.split('/').first
+        model_name = segments.first.classify
+        record_id  = segments[2..-2].join.to_i
+
+        next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
+
+        preload_map[model_name] << record_id
+      end
+
+      preload_map.each_with_object({}) do |(model_name, record_ids), model_map|
+        model_map[model_name] = model_name.constantize.where(id: record_ids).each_with_object({}) { |record, record_map| record_map[record.id] = record }
       end
     end
   end
diff --git a/lib/mastodon/statuses_cli.rb b/lib/mastodon/statuses_cli.rb
index 875183372..8a18a3b2f 100644
--- a/lib/mastodon/statuses_cli.rb
+++ b/lib/mastodon/statuses_cli.rb
@@ -14,6 +14,7 @@ module Mastodon
 
     option :days, type: :numeric, default: 90
     option :clean_followed, type: :boolean
+    option :skip_media_remove, type: :boolean
     desc 'remove', 'Remove unreferenced statuses'
     long_desc <<~LONG_DESC
       Remove statuses that are not referenced by local user activity, such as
@@ -58,9 +59,10 @@ module Mastodon
 
       scope.in_batches.delete_all
 
-      say('Beginning removal of now-orphaned media attachments to free up disk space...')
-
-      Scheduler::MediaCleanupScheduler.new.perform
+      unless options[:skip_media_remove]
+        say('Beginning removal of now-orphaned media attachments to free up disk space...')
+        Scheduler::MediaCleanupScheduler.new.perform
+      end
 
       say("Done after #{Time.now.to_f - start_at}s", :green)
     ensure
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 49fe73043..ba78a3c30 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      2
+      3
     end
 
     def flags
diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb
new file mode 100644
index 000000000..d9ec0159a
--- /dev/null
+++ b/lib/paperclip/attachment_extensions.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Paperclip
+  module AttachmentExtensions
+    # We overwrite this method to support delayed processing in
+    # Sidekiq. Since we process the original file to reduce disk
+    # usage, and we still want to generate thumbnails straight
+    # away, it's the only style we need to exclude
+    def process_style?(style_name, style_args)
+      if style_name == :original && instance.respond_to?(:delay_processing?) && instance.delay_processing?
+        false
+      else
+        style_args.empty? || style_args.include?(style_name)
+      end
+    end
+
+    def reprocess_original!
+      old_original_path = path(:original)
+      reprocess!(:original)
+      new_original_path = path(:original)
+
+      if new_original_path != old_original_path
+        @queued_for_delete << old_original_path
+        flush_deletes
+      end
+    end
+
+    def variant?(other_filename)
+      return true  if original_filename == other_filename
+      return false if original_filename.nil?
+
+      formats = styles.values.map(&:format).compact
+
+      return false if formats.empty?
+
+      other_extension = File.extname(other_filename)
+
+      formats.include?(other_extension.delete('.')) && File.basename(other_filename, other_extension) == File.basename(original_filename, File.extname(original_filename))
+    end
+  end
+end
+
+Paperclip::Attachment.prepend(Paperclip::AttachmentExtensions)
diff --git a/lib/paperclip/url_generator_extensions.rb b/lib/paperclip/url_generator_extensions.rb
new file mode 100644
index 000000000..1079efdbc
--- /dev/null
+++ b/lib/paperclip/url_generator_extensions.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Paperclip
+  module UrlGeneratorExtensions
+    # Monkey-patch Paperclip to use Addressable::URI's normalization instead
+    # of the long-deprecated URI.esacpe
+    def escape_url(url)
+      if url.respond_to?(:escape)
+        url.escape
+      else
+        Addressable::URI.parse(url).normalize.to_str.gsub(escape_regex) { |m| "%#{m.ord.to_s(16).upcase}" }
+      end
+    end
+  end
+end
+
+Paperclip::UrlGenerator.prepend(Paperclip::UrlGeneratorExtensions)
diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb
index 66f7feda5..4d9544231 100644
--- a/lib/paperclip/video_transcoder.rb
+++ b/lib/paperclip/video_transcoder.rb
@@ -5,12 +5,22 @@ module Paperclip
   # to check when uploaded videos are actually gifv's
   class VideoTranscoder < Paperclip::Processor
     def make
-      meta = ::Av.cli.identify(@file.path)
+      movie = FFMPEG::Movie.new(@file.path)
 
-      attachment.instance.type = MediaAttachment.types[:gifv] unless meta[:audio_encode]
-      options[:format] = File.extname(attachment.instance.file_file_name)[1..-1] if options[:keep_same_format]
+      attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec
 
-      Paperclip::Transcoder.make(file, options, attachment)
+      Paperclip::Transcoder.make(file, actual_options(movie), attachment)
+    end
+
+    private
+
+    def actual_options(movie)
+      opts = options[:passthrough_options]
+      if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace)
+        opts[:options]
+      else
+        options
+      end
     end
   end
 end