Add customizable thumbnails for audio and video attachments (#14145)

- Change audio files to not be stripped of metadata
- Automatically extract cover art from audio if it exists
- Add `thumbnail` parameter to `POST /api/v1/media`, `POST /api/v2/media` and `PUT /api/v1/media/:id`
- Add `icon` to represent it in attachments in ActivityPub
- Fix `preview_url` containing URL of missing missing image when there is no thumbnail instead of null
- Fix duration of audio not being displayed on public pages until the file is loaded

2 files changed, 86 insertions, 51 deletions

diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb
index c728a460e..6fc1dcc26 100644
--- a/app/models/concerns/remotable.rb
+++ b/app/models/concerns/remotable.rb
@@ -4,12 +4,12 @@ module Remotable
   extend ActiveSupport::Concern
 
   class_methods do
-    def remotable_attachment(attachment_name, limit, suppress_errors: true)
-      attribute_name  = "#{attachment_name}_remote_url".to_sym
-      method_name     = "#{attribute_name}=".to_sym
-      alt_method_name = "reset_#{attachment_name}!".to_sym
+    def remotable_attachment(attachment_name, limit, suppress_errors: true, download_on_assign: true, attribute_name: nil)
+      attribute_name ||= "#{attachment_name}_remote_url".to_sym
+
+      define_method("download_#{attachment_name}!") do
+        url = self[attribute_name]
 
-      define_method method_name do |url|
         return if url.blank?
 
         begin
@@ -18,7 +18,7 @@ module Remotable
           return
         end
 
-        return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || (self[attribute_name] == url && send("#{attachment_name}_file_name").present?)
+        return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank?
 
         begin
           Request.new(:get, url).perform do |response|
@@ -36,10 +36,8 @@ module Remotable
 
             basename = SecureRandom.hex(8)
 
-            send("#{attachment_name}_file_name=", basename + extname)
-            send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
-
-            self[attribute_name] = url if has_attribute?(attribute_name)
+            public_send("#{attachment_name}_file_name=", basename + extname)
+            public_send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
           end
         rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e
           Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
@@ -50,14 +48,15 @@ module Remotable
         end
       end
 
-      define_method alt_method_name do
-        url = self[attribute_name]
+      define_method("#{attribute_name}=") do |url|
+        return if self[attribute_name] == url && public_send("#{attachment_name}_file_name").present?
 
-        return if url.blank?
+        self[attribute_name] = url
 
-        self[attribute_name] = ''
-        send(method_name, url)
+        public_send("download_#{attachment_name}!") if download_on_assign
       end
+
+      alias_method("reset_#{attachment_name}!", "download_#{attachment_name}!")
     end
   end
 
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index d44467009..f67566a18 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -21,6 +21,11 @@
 #  blurhash                    :string
 #  processing                  :integer
 #  file_storage_schema_version :integer
+#  thumbnail_file_name         :string
+#  thumbnail_content_type      :string
+#  thumbnail_file_size         :integer
+#  thumbnail_updated_at        :datetime
+#  thumbnail_remote_url        :string
 #
 
 class MediaAttachment < ApplicationRecord
@@ -49,13 +54,13 @@ class MediaAttachment < ApplicationRecord
     original: {
       pixels: 1_638_400, # 1280x1280px
       file_geometry_parser: FastGeometryParser,
-    },
+    }.freeze,
 
     small: {
       pixels: 160_000, # 400x400px
       file_geometry_parser: FastGeometryParser,
       blurhash: BLURHASH_OPTIONS,
-    },
+    }.freeze,
   }.freeze
 
   VIDEO_FORMAT = {
@@ -74,14 +79,14 @@ class MediaAttachment < ApplicationRecord
         'frames:v' => 60 * 60 * 3,
         'crf' => 18,
         'map_metadata' => '-1',
-      },
-    },
+      }.freeze,
+    }.freeze,
   }.freeze
 
   VIDEO_PASSTHROUGH_OPTIONS = {
-    video_codecs: ['h264'],
-    audio_codecs: ['aac', nil],
-    colorspaces: ['yuv420p'],
+    video_codecs: ['h264'].freeze,
+    audio_codecs: ['aac', nil].freeze,
+    colorspaces: ['yuv420p'].freeze,
     options: {
       format: 'mp4',
       convert_options: {
@@ -90,9 +95,9 @@ class MediaAttachment < ApplicationRecord
           'map_metadata' => '-1',
           'c:v' => 'copy',
           'c:a' => 'copy',
-        },
-      },
-    },
+        }.freeze,
+      }.freeze,
+    }.freeze,
   }.freeze
 
   VIDEO_STYLES = {
@@ -101,15 +106,15 @@ class MediaAttachment < ApplicationRecord
         output: {
           'loglevel' => 'fatal',
           vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
-        },
-      },
+        }.freeze,
+      }.freeze,
       format: 'png',
       time: 0,
       file_geometry_parser: FastGeometryParser,
       blurhash: BLURHASH_OPTIONS,
-    },
+    }.freeze,
 
-    original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS),
+    original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS).freeze,
   }.freeze
 
   AUDIO_STYLES = {
@@ -119,16 +124,23 @@ class MediaAttachment < ApplicationRecord
       convert_options: {
         output: {
           'loglevel' => 'fatal',
-          'map_metadata' => '-1',
           'q:a' => 2,
-        },
-      },
-    },
+        }.freeze,
+      }.freeze,
+    }.freeze,
   }.freeze
 
   VIDEO_CONVERTED_STYLES = {
-    small: VIDEO_STYLES[:small],
-    original: VIDEO_FORMAT,
+    small: VIDEO_STYLES[:small].freeze,
+    original: VIDEO_FORMAT.freeze,
+  }.freeze
+
+  THUMBNAIL_STYLES = {
+    original: IMAGE_STYLES[:small].freeze,
+  }.freeze
+
+  GLOBAL_CONVERT_OPTIONS = {
+    all: '-quality 90 -strip +set modify-date +set create-date',
   }.freeze
 
   IMAGE_LIMIT = 10.megabytes
@@ -144,18 +156,28 @@ class MediaAttachment < ApplicationRecord
   has_attached_file :file,
                     styles: ->(f) { file_styles f },
                     processors: ->(f) { file_processors f },
-                    convert_options: { all: '-quality 90 -strip +set modify-date +set create-date' }
+                    convert_options: GLOBAL_CONVERT_OPTIONS
 
   validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
   validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
   validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
-  remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false
+  remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false, download_on_assign: false, attribute_name: :remote_url
+
+  has_attached_file :thumbnail,
+                    styles: THUMBNAIL_STYLES,
+                    processors: [:lazy_thumbnail, :blurhash_transcoder],
+                    convert_options: GLOBAL_CONVERT_OPTIONS
+
+  validates_attachment_content_type :thumbnail, content_type: IMAGE_MIME_TYPES
+  validates_attachment_size :thumbnail, less_than: IMAGE_LIMIT
+  remotable_attachment :thumbnail, IMAGE_LIMIT, suppress_errors: true, download_on_assign: false
 
   include Attachmentable
 
   validates :account, presence: true
   validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }, if: :local?
   validates :file, presence: true, if: :local?
+  validates :thumbnail, absence: true, if: -> { local? && !audio_or_video? }
 
   scope :attached,   -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) }
   scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
@@ -215,16 +237,21 @@ class MediaAttachment < ApplicationRecord
     @delay_processing
   end
 
+  def delay_processing_for_attachment?(attachment_name)
+    @delay_processing && attachment_name == :file
+  end
+
   after_commit :enqueue_processing, on: :create
   after_commit :reset_parent_cache, on: :update
 
   before_create :prepare_description, unless: :local?
   before_create :set_shortcode
   before_create :set_processing
-  before_create :set_meta
 
-  before_post_process :set_type_and_extension
-  before_post_process :check_video_dimensions
+  after_post_process :set_meta
+
+  before_file_post_process :set_type_and_extension
+  before_file_post_process :check_video_dimensions
 
   class << self
     def supported_mime_types
@@ -237,25 +264,25 @@ class MediaAttachment < ApplicationRecord
 
     private
 
-    def file_styles(f)
-      if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
+    def file_styles(attachment)
+      if attachment.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(attachment.instance.file_content_type)
         VIDEO_CONVERTED_STYLES
-      elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type)
+      elsif IMAGE_MIME_TYPES.include?(attachment.instance.file_content_type)
         IMAGE_STYLES
-      elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type)
+      elsif VIDEO_MIME_TYPES.include?(attachment.instance.file_content_type)
         VIDEO_STYLES
       else
         AUDIO_STYLES
       end
     end
 
-    def file_processors(f)
-      if f.file_content_type == 'image/gif'
+    def file_processors(instance)
+      if instance.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
-      elsif VIDEO_MIME_TYPES.include?(f.file_content_type)
+      elsif VIDEO_MIME_TYPES.include?(instance.file_content_type)
         [:video_transcoder, :blurhash_transcoder, :type_corrector]
-      elsif AUDIO_MIME_TYPES.include?(f.file_content_type)
-        [:transcoder, :type_corrector]
+      elsif AUDIO_MIME_TYPES.include?(instance.file_content_type)
+        [:image_extractor, :transcoder, :type_corrector]
       else
         [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
       end
@@ -298,7 +325,7 @@ class MediaAttachment < ApplicationRecord
   def check_video_dimensions
     return unless (video? || gifv?) && file.queued_for_write[:original].present?
 
-    movie = FFMPEG::Movie.new(file.queued_for_write[:original].path)
+    movie = ffmpeg_data(file.queued_for_write[:original].path)
 
     return unless movie.valid?
 
@@ -317,6 +344,8 @@ class MediaAttachment < ApplicationRecord
       meta[style] = style == :small || image? ? image_geometry(file) : video_metadata(file)
     end
 
+    meta[:small] = image_geometry(thumbnail.queued_for_write[:original]) if thumbnail.queued_for_write.key?(:original)
+
     meta
   end
 
@@ -334,7 +363,7 @@ class MediaAttachment < ApplicationRecord
   end
 
   def video_metadata(file)
-    movie = FFMPEG::Movie.new(file.path)
+    movie = ffmpeg_data(file.path)
 
     return {} unless movie.valid?
 
@@ -347,6 +376,13 @@ class MediaAttachment < ApplicationRecord
     }.compact
   end
 
+  # We call this method about 3 different times on potentially different
+  # paths but ultimately the same file, so it makes sense to memoize the
+  # result while disregarding the path
+  def ffmpeg_data(path = nil)
+    @ffmpeg_data ||= FFMPEG::Movie.new(path)
+  end
+
   def enqueue_processing
     PostProcessMediaWorker.perform_async(id) if delay_processing?
   end