From 64aac3073340dbc92c33f5f1c6f76dcafa77a450 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 29 Jun 2020 13:56:55 +0200 Subject: Add customizable thumbnails for audio and video attachments (#14145) - Change audio files to not be stripped of metadata - Automatically extract cover art from audio if it exists - Add `thumbnail` parameter to `POST /api/v1/media`, `POST /api/v2/media` and `PUT /api/v1/media/:id` - Add `icon` to represent it in attachments in ActivityPub - Fix `preview_url` containing URL of missing missing image when there is no thumbnail instead of null - Fix duration of audio not being displayed on public pages until the file is loaded --- app/models/concerns/remotable.rb | 29 +++++------ app/models/media_attachment.rb | 108 ++++++++++++++++++++++++++------------- 2 files changed, 86 insertions(+), 51 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index c728a460e..6fc1dcc26 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -4,12 +4,12 @@ module Remotable extend ActiveSupport::Concern class_methods do - def remotable_attachment(attachment_name, limit, suppress_errors: true) - attribute_name = "#{attachment_name}_remote_url".to_sym - method_name = "#{attribute_name}=".to_sym - alt_method_name = "reset_#{attachment_name}!".to_sym + def remotable_attachment(attachment_name, limit, suppress_errors: true, download_on_assign: true, attribute_name: nil) + attribute_name ||= "#{attachment_name}_remote_url".to_sym + + define_method("download_#{attachment_name}!") do + url = self[attribute_name] - define_method method_name do |url| return if url.blank? begin @@ -18,7 +18,7 @@ module Remotable return end - return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || (self[attribute_name] == url && send("#{attachment_name}_file_name").present?) + return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? begin Request.new(:get, url).perform do |response| @@ -36,10 +36,8 @@ module Remotable basename = SecureRandom.hex(8) - send("#{attachment_name}_file_name=", basename + extname) - send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit))) - - self[attribute_name] = url if has_attribute?(attribute_name) + public_send("#{attachment_name}_file_name=", basename + extname) + public_send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit))) end rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" @@ -50,14 +48,15 @@ module Remotable end end - define_method alt_method_name do - url = self[attribute_name] + define_method("#{attribute_name}=") do |url| + return if self[attribute_name] == url && public_send("#{attachment_name}_file_name").present? - return if url.blank? + self[attribute_name] = url - self[attribute_name] = '' - send(method_name, url) + public_send("download_#{attachment_name}!") if download_on_assign end + + alias_method("reset_#{attachment_name}!", "download_#{attachment_name}!") end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index d44467009..f67566a18 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -21,6 +21,11 @@ # blurhash :string # processing :integer # file_storage_schema_version :integer +# thumbnail_file_name :string +# thumbnail_content_type :string +# thumbnail_file_size :integer +# thumbnail_updated_at :datetime +# thumbnail_remote_url :string # class MediaAttachment < ApplicationRecord @@ -49,13 +54,13 @@ class MediaAttachment < ApplicationRecord original: { pixels: 1_638_400, # 1280x1280px file_geometry_parser: FastGeometryParser, - }, + }.freeze, small: { pixels: 160_000, # 400x400px file_geometry_parser: FastGeometryParser, blurhash: BLURHASH_OPTIONS, - }, + }.freeze, }.freeze VIDEO_FORMAT = { @@ -74,14 +79,14 @@ class MediaAttachment < ApplicationRecord 'frames:v' => 60 * 60 * 3, 'crf' => 18, 'map_metadata' => '-1', - }, - }, + }.freeze, + }.freeze, }.freeze VIDEO_PASSTHROUGH_OPTIONS = { - video_codecs: ['h264'], - audio_codecs: ['aac', nil], - colorspaces: ['yuv420p'], + video_codecs: ['h264'].freeze, + audio_codecs: ['aac', nil].freeze, + colorspaces: ['yuv420p'].freeze, options: { format: 'mp4', convert_options: { @@ -90,9 +95,9 @@ class MediaAttachment < ApplicationRecord 'map_metadata' => '-1', 'c:v' => 'copy', 'c:a' => 'copy', - }, - }, - }, + }.freeze, + }.freeze, + }.freeze, }.freeze VIDEO_STYLES = { @@ -101,15 +106,15 @@ class MediaAttachment < ApplicationRecord output: { 'loglevel' => 'fatal', vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease', - }, - }, + }.freeze, + }.freeze, format: 'png', time: 0, file_geometry_parser: FastGeometryParser, blurhash: BLURHASH_OPTIONS, - }, + }.freeze, - original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS), + original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS).freeze, }.freeze AUDIO_STYLES = { @@ -119,16 +124,23 @@ class MediaAttachment < ApplicationRecord convert_options: { output: { 'loglevel' => 'fatal', - 'map_metadata' => '-1', 'q:a' => 2, - }, - }, - }, + }.freeze, + }.freeze, + }.freeze, }.freeze VIDEO_CONVERTED_STYLES = { - small: VIDEO_STYLES[:small], - original: VIDEO_FORMAT, + small: VIDEO_STYLES[:small].freeze, + original: VIDEO_FORMAT.freeze, + }.freeze + + THUMBNAIL_STYLES = { + original: IMAGE_STYLES[:small].freeze, + }.freeze + + GLOBAL_CONVERT_OPTIONS = { + all: '-quality 90 -strip +set modify-date +set create-date', }.freeze IMAGE_LIMIT = 10.megabytes @@ -144,18 +156,28 @@ class MediaAttachment < ApplicationRecord has_attached_file :file, styles: ->(f) { file_styles f }, processors: ->(f) { file_processors f }, - convert_options: { all: '-quality 90 -strip +set modify-date +set create-date' } + convert_options: GLOBAL_CONVERT_OPTIONS validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format? - remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false + remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false, download_on_assign: false, attribute_name: :remote_url + + has_attached_file :thumbnail, + styles: THUMBNAIL_STYLES, + processors: [:lazy_thumbnail, :blurhash_transcoder], + convert_options: GLOBAL_CONVERT_OPTIONS + + validates_attachment_content_type :thumbnail, content_type: IMAGE_MIME_TYPES + validates_attachment_size :thumbnail, less_than: IMAGE_LIMIT + remotable_attachment :thumbnail, IMAGE_LIMIT, suppress_errors: true, download_on_assign: false include Attachmentable validates :account, presence: true validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }, if: :local? validates :file, presence: true, if: :local? + validates :thumbnail, absence: true, if: -> { local? && !audio_or_video? } scope :attached, -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) } scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) } @@ -215,16 +237,21 @@ class MediaAttachment < ApplicationRecord @delay_processing end + def delay_processing_for_attachment?(attachment_name) + @delay_processing && attachment_name == :file + end + after_commit :enqueue_processing, on: :create after_commit :reset_parent_cache, on: :update before_create :prepare_description, unless: :local? before_create :set_shortcode before_create :set_processing - before_create :set_meta - before_post_process :set_type_and_extension - before_post_process :check_video_dimensions + after_post_process :set_meta + + before_file_post_process :set_type_and_extension + before_file_post_process :check_video_dimensions class << self def supported_mime_types @@ -237,25 +264,25 @@ class MediaAttachment < ApplicationRecord private - def file_styles(f) - if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type) + def file_styles(attachment) + if attachment.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(attachment.instance.file_content_type) VIDEO_CONVERTED_STYLES - elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type) + elsif IMAGE_MIME_TYPES.include?(attachment.instance.file_content_type) IMAGE_STYLES - elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type) + elsif VIDEO_MIME_TYPES.include?(attachment.instance.file_content_type) VIDEO_STYLES else AUDIO_STYLES end end - def file_processors(f) - if f.file_content_type == 'image/gif' + def file_processors(instance) + if instance.file_content_type == 'image/gif' [:gif_transcoder, :blurhash_transcoder] - elsif VIDEO_MIME_TYPES.include?(f.file_content_type) + elsif VIDEO_MIME_TYPES.include?(instance.file_content_type) [:video_transcoder, :blurhash_transcoder, :type_corrector] - elsif AUDIO_MIME_TYPES.include?(f.file_content_type) - [:transcoder, :type_corrector] + elsif AUDIO_MIME_TYPES.include?(instance.file_content_type) + [:image_extractor, :transcoder, :type_corrector] else [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] end @@ -298,7 +325,7 @@ class MediaAttachment < ApplicationRecord def check_video_dimensions return unless (video? || gifv?) && file.queued_for_write[:original].present? - movie = FFMPEG::Movie.new(file.queued_for_write[:original].path) + movie = ffmpeg_data(file.queued_for_write[:original].path) return unless movie.valid? @@ -317,6 +344,8 @@ class MediaAttachment < ApplicationRecord meta[style] = style == :small || image? ? image_geometry(file) : video_metadata(file) end + meta[:small] = image_geometry(thumbnail.queued_for_write[:original]) if thumbnail.queued_for_write.key?(:original) + meta end @@ -334,7 +363,7 @@ class MediaAttachment < ApplicationRecord end def video_metadata(file) - movie = FFMPEG::Movie.new(file.path) + movie = ffmpeg_data(file.path) return {} unless movie.valid? @@ -347,6 +376,13 @@ class MediaAttachment < ApplicationRecord }.compact end + # We call this method about 3 different times on potentially different + # paths but ultimately the same file, so it makes sense to memoize the + # result while disregarding the path + def ffmpeg_data(path = nil) + @ffmpeg_data ||= FFMPEG::Movie.new(path) + end + def enqueue_processing PostProcessMediaWorker.perform_async(id) if delay_processing? end -- cgit