From 9660aa4543deff41c60d131e081137f84e771499 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 8 Mar 2020 23:56:18 +0100 Subject: Change local media attachments to perform heavy processing asynchronously (#13210) Fix #9106 --- lib/paperclip/attachment_extensions.rb | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 lib/paperclip/attachment_extensions.rb (limited to 'lib/paperclip') diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb new file mode 100644 index 000000000..3b308af5f --- /dev/null +++ b/lib/paperclip/attachment_extensions.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Paperclip + module AttachmentExtensions + # We overwrite this method to support delayed processing in + # Sidekiq. Since we process the original file to reduce disk + # usage, and we still want to generate thumbnails straight + # away, it's the only style we need to exclude + def process_style?(style_name, style_args) + if style_name == :original && instance.respond_to?(:delay_processing?) && instance.delay_processing? + false + else + style_args.empty? || style_args.include?(style_name) + end + end + + def reprocess_original! + old_original_path = path(:original) + reprocess!(:original) + new_original_path = path(:original) + + if new_original_path != old_original_path + @queued_for_delete << old_original_path + flush_deletes + end + end + end +end + +Paperclip::Attachment.prepend(Paperclip::AttachmentExtensions) -- cgit From abd839488054b6de2dec9e7ec095d79e4a106573 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 9 Mar 2020 23:15:59 +0100 Subject: Fix MP4 (H264 + AAC) video files being needlessly re-encoded (#13239) --- app/models/media_attachment.rb | 18 +++++++++++++++++- lib/paperclip/video_transcoder.rb | 16 ++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) (limited to 'lib/paperclip') diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 7c9b4b909..9ca0a6cda 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -77,6 +77,22 @@ class MediaAttachment < ApplicationRecord }, }.freeze + VIDEO_PASSTHROUGH_OPTIONS = { + video_codec_whitelist: ['h264'], + audio_codec_whitelist: ['aac', nil], + options: { + format: 'mp4', + convert_options: { + output: { + 'loglevel' => 'fatal', + 'map_metadata' => '-1', + 'c:v' => 'copy', + 'c:a' => 'copy', + }, + }, + }, + }.freeze + VIDEO_STYLES = { small: { convert_options: { @@ -91,7 +107,7 @@ class MediaAttachment < ApplicationRecord blurhash: BLURHASH_OPTIONS, }, - original: VIDEO_FORMAT, + original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS), }.freeze AUDIO_STYLES = { diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb index 66f7feda5..0de548964 100644 --- a/lib/paperclip/video_transcoder.rb +++ b/lib/paperclip/video_transcoder.rb @@ -5,12 +5,20 @@ module Paperclip # to check when uploaded videos are actually gifv's class VideoTranscoder < Paperclip::Processor def make - meta = ::Av.cli.identify(@file.path) + movie = FFMPEG::Movie.new(@file.path) + actual_options = options + passthrough_options = actual_options[:passthrough_options] + actual_options = passthrough_options[:options] if passthrough?(movie, passthrough_options) - attachment.instance.type = MediaAttachment.types[:gifv] unless meta[:audio_encode] - options[:format] = File.extname(attachment.instance.file_file_name)[1..-1] if options[:keep_same_format] + attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec - Paperclip::Transcoder.make(file, options, attachment) + Paperclip::Transcoder.make(file, actual_options, attachment) + end + + private + + def passthrough?(movie, options) + options && options[:video_codec_whitelist].include?(movie.video_codec) && options[:audio_codec_whitelist].include?(movie.audio_codec) end end end -- cgit From 2c6099125d9ed5c2add62e84d5ba7b93de658910 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 10 Mar 2020 11:58:40 +0100 Subject: Fix videos with unsupported colorspace not being transcoded (#13242) --- app/models/media_attachment.rb | 5 +++-- lib/paperclip/video_transcoder.rb | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) (limited to 'lib/paperclip') diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 9ca0a6cda..31aa918b7 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -78,8 +78,9 @@ class MediaAttachment < ApplicationRecord }.freeze VIDEO_PASSTHROUGH_OPTIONS = { - video_codec_whitelist: ['h264'], - audio_codec_whitelist: ['aac', nil], + video_codecs: ['h264'], + audio_codecs: ['aac', nil], + colorspaces: ['yuv420p'], options: { format: 'mp4', convert_options: { diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb index 0de548964..4d9544231 100644 --- a/lib/paperclip/video_transcoder.rb +++ b/lib/paperclip/video_transcoder.rb @@ -6,19 +6,21 @@ module Paperclip class VideoTranscoder < Paperclip::Processor def make movie = FFMPEG::Movie.new(@file.path) - actual_options = options - passthrough_options = actual_options[:passthrough_options] - actual_options = passthrough_options[:options] if passthrough?(movie, passthrough_options) attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec - Paperclip::Transcoder.make(file, actual_options, attachment) + Paperclip::Transcoder.make(file, actual_options(movie), attachment) end private - def passthrough?(movie, options) - options && options[:video_codec_whitelist].include?(movie.video_codec) && options[:audio_codec_whitelist].include?(movie.audio_codec) + def actual_options(movie) + opts = options[:passthrough_options] + if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace) + opts[:options] + else + options + end end end end -- cgit From 0c8945e5ff9e75dd322ee2b2c4ea9cc1dc728911 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 26 Mar 2020 01:56:41 +0100 Subject: Change `tootctl media remove-orphans` to work for all classes (#13316) Change `tootctl media lookup` to not use an interactive prompt --- app/lib/activitypub/tag_manager.rb | 2 + app/models/media_attachment.rb | 13 ---- lib/mastodon/media_cli.rb | 111 ++++++++++++++++++++++++--------- lib/paperclip/attachment_extensions.rb | 13 ++++ 4 files changed, 98 insertions(+), 41 deletions(-) (limited to 'lib/paperclip') diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index ed680d762..1523f86d4 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -15,6 +15,8 @@ class ActivityPub::TagManager def url_for(target) return target.url if target.respond_to?(:local?) && !target.local? + return unless target.respond_to?(:object_type) + case target.object_type when :person target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 31aa918b7..f45e2c9f7 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -184,19 +184,6 @@ class MediaAttachment < ApplicationRecord audio? || video? end - def variant?(other_file_name) - return true if file_file_name == other_file_name - return false if file_file_name.nil? - - formats = file.styles.values.map(&:format).compact - - return false if formats.empty? - - extension = File.extname(other_file_name) - - formats.include?(extension.delete('.')) && File.basename(other_file_name, extension) == File.basename(file_file_name, File.extname(file_file_name)) - end - def to_param shortcode end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index d842b986f..b4ad78fe5 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -45,6 +45,7 @@ module Mastodon end option :start_after + option :prefix option :dry_run, type: :boolean, default: false desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments' long_desc <<~LONG_DESC @@ -58,6 +59,7 @@ module Mastodon reclaimed_bytes = 0 removed = 0 dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + prefix = options[:prefix] case Paperclip::Attachment.default_options[:storage] when :s3 @@ -69,7 +71,7 @@ module Mastodon loop do objects = begin begin - bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x } + bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x } rescue => e progress.log(pastel.red("Error fetching list of files: #{e}")) progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key @@ -79,16 +81,21 @@ module Mastodon break if objects.empty? - last_key = objects.last.key - attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment } + last_key = objects.last.key + record_map = preload_records_from_mixed_objects(objects) objects.each do |object| - attachment_id = object.key.split('/')[2..-2].join.to_i - filename = object.key.split('/').last + path_segments = object.key.split('/') + model_name = path_segments.first.classify + attachment_name = path_segments[1].singularize + record_id = path_segments[2..-2].join.to_i + file_name = path_segments.last + record = record_map.dig(model_name, record_id) + attachment = record&.public_send(attachment_name) progress.increment - next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename) + next unless attachment.blank? || !attachment.variant?(file_name) begin object.delete unless options[:dry_run] @@ -110,17 +117,24 @@ module Mastodon root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s) - Find.find(File.join(root_path, 'media_attachments', 'files')) do |path| + Find.find(File.join(*[root_path, prefix].compact)) do |path| next if File.directory?(path) - key = path.gsub("#{root_path}#{File::SEPARATOR}", '') - attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i - filename = key.split(File::SEPARATOR).last - attachment = MediaAttachment.find_by(id: attachment_id) + key = path.gsub("#{root_path}#{File::SEPARATOR}", '') + path_segments = key.split(File::SEPARATOR) + model_name = path_segments.first.classify + record_id = path_segments[2..-2].join.to_i + attachment_name = path_segments[1].singularize + file_name = path_segments.last + + next unless PRELOAD_MODEL_WHITELIST.include?(model_name) + + record = model_name.constantize.find_by(id: record_id) + attachment = record&.public_send(attachment_name) progress.increment - next unless attachment.nil? || !attachment.variant?(filename) + next unless attachment.blank? || !attachment.variant?(file_name) begin size = File.size(path) @@ -213,25 +227,66 @@ module Mastodon say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}") end - desc 'lookup', 'Lookup where media is displayed by passing a media URL' - def lookup - prompt = TTY::Prompt.new + desc 'lookup URL', 'Lookup where media is displayed by passing a media URL' + def lookup(url) + path = Addressable::URI.parse(url).path + path_segments = path.split('/')[2..-1] + model_name = path_segments.first.classify + record_id = path_segments[2..-2].join.to_i - url = prompt.ask('Please enter a URL to the media to lookup:', required: true) + unless PRELOAD_MODEL_WHITELIST.include?(model_name) + say("Cannot find corresponding model: #{model_name}", :red) + exit(1) + end - attachment_id = url - .split('/')[0..-2] - .grep(/\A\d+\z/) - .join('') + record = model_name.constantize.find_by(id: record_id) + record = record.status if record.respond_to?(:status) - if url.split('/')[0..-2].include? 'media_attachments' - model = MediaAttachment.find(attachment_id).status - prompt.say(ActivityPub::TagManager.instance.url_for(model)) - elsif url.split('/')[0..-2].include? 'accounts' - model = Account.find(attachment_id) - prompt.say(ActivityPub::TagManager.instance.url_for(model)) - else - prompt.say('Not found') + unless record + say('Cannot find corresponding record', :red) + exit(1) + end + + display_url = ActivityPub::TagManager.instance.url_for(record) + + if display_url.blank? + say('No public URL for this type of record', :red) + exit(1) + end + + say(display_url, :blue) + rescue Addressable::URI::InvalidURIError + say('Invalid URL', :red) + exit(1) + end + + private + + PRELOAD_MODEL_WHITELIST = %w( + Account + Backup + CustomEmoji + Import + MediaAttachment + PreviewCard + SiteUpload + ).freeze + + def preload_records_from_mixed_objects(objects) + preload_map = Hash.new { |hash, key| hash[key] = [] } + + objects.map do |object| + segments = object.key.split('/').first + model_name = segments.first.classify + record_id = segments[2..-2].join.to_i + + next unless PRELOAD_MODEL_WHITELIST.include?(model_name) + + preload_map[model_name] << record_id + end + + preload_map.each_with_object({}) do |(model_name, record_ids), model_map| + model_map[model_name] = model_name.constantize.where(id: record_ids).each_with_object({}) { |record, record_map| record_map[record.id] = record } end end end diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index 3b308af5f..d9ec0159a 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -24,6 +24,19 @@ module Paperclip flush_deletes end end + + def variant?(other_filename) + return true if original_filename == other_filename + return false if original_filename.nil? + + formats = styles.values.map(&:format).compact + + return false if formats.empty? + + other_extension = File.extname(other_filename) + + formats.include?(other_extension.delete('.')) && File.basename(other_filename, other_extension) == File.basename(original_filename, File.extname(original_filename)) + end end end -- cgit From 6c79b7237e31eb510af7df3f4f2cb133dea39845 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 26 Mar 2020 15:09:16 +0100 Subject: Fix Paperclip using deprecated URI.escape function (#13320) Monkey-patch Paperclip to perform URL escaping in a slightly more appropriate way, and get rid of runtime deprecation warnings. --- config/application.rb | 1 + lib/paperclip/url_generator_extensions.rb | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 lib/paperclip/url_generator_extensions.rb (limited to 'lib/paperclip') diff --git a/config/application.rb b/config/application.rb index cd599eefc..4c34efa15 100644 --- a/config/application.rb +++ b/config/application.rb @@ -7,6 +7,7 @@ require 'rails/all' Bundler.require(*Rails.groups) require_relative '../app/lib/exceptions' +require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/gif_transcoder' diff --git a/lib/paperclip/url_generator_extensions.rb b/lib/paperclip/url_generator_extensions.rb new file mode 100644 index 000000000..1079efdbc --- /dev/null +++ b/lib/paperclip/url_generator_extensions.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Paperclip + module UrlGeneratorExtensions + # Monkey-patch Paperclip to use Addressable::URI's normalization instead + # of the long-deprecated URI.esacpe + def escape_url(url) + if url.respond_to?(:escape) + url.escape + else + Addressable::URI.parse(url).normalize.to_str.gsub(escape_regex) { |m| "%#{m.ord.to_s(16).upcase}" } + end + end + end +end + +Paperclip::UrlGenerator.prepend(Paperclip::UrlGeneratorExtensions) -- cgit