diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/exceptions.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/migration_helpers.rb | 16 | ||||
| -rw-r--r-- | lib/mastodon/redis_config.rb | 2 | ||||
| -rw-r--r-- | lib/mastodon/search_cli.rb | 4 | ||||
| -rw-r--r-- | lib/paperclip/attachment_extensions.rb | 4 | ||||
| -rw-r--r-- | lib/paperclip/gif_transcoder.rb | 3 | ||||
| -rw-r--r-- | lib/paperclip/image_extractor.rb | 14 | ||||
| -rw-r--r-- | lib/paperclip/schema_extensions.rb | 37 | ||||
| -rw-r--r-- | lib/paperclip/transcoder.rb | 102 | ||||
| -rw-r--r-- | lib/paperclip/transcoder_extensions.rb | 14 | ||||
| -rw-r--r-- | lib/paperclip/validation_extensions.rb | 58 | ||||
| -rw-r--r-- | lib/paperclip/video_transcoder.rb | 26 | ||||
| -rw-r--r-- | lib/tasks/db.rake | 2 | ||||
| -rw-r--r-- | lib/terrapin/multi_pipe_extensions.rb | 66 |
14 files changed, 291 insertions, 61 deletions
diff --git a/lib/exceptions.rb b/lib/exceptions.rb index 7c8e77871..eb472abaa 100644 --- a/lib/exceptions.rb +++ b/lib/exceptions.rb @@ -12,7 +12,11 @@ module Mastodon class RateLimitExceededError < Error; end class UnexpectedResponseError < Error + attr_reader :response + def initialize(response = nil) + @response = response + if response.respond_to? :uri super("#{response.uri} returned code #{response.code}") else diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 147642a1c..521d903bf 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -95,7 +95,7 @@ module Mastodon allow_null: options[:null] ) else - add_column(table_name, column_name, :datetime_with_timezone, options) + add_column(table_name, column_name, :datetime_with_timezone, **options) end end end @@ -120,7 +120,7 @@ module Mastodon options = options.merge({ algorithm: :concurrently }) disable_statement_timeout - add_index(table_name, column_name, options) + add_index(table_name, column_name, **options) end # Removes an existed index, concurrently when supported @@ -144,7 +144,7 @@ module Mastodon disable_statement_timeout end - remove_index(table_name, options.merge({ column: column_name })) + remove_index(table_name, **options.merge({ column: column_name })) end # Removes an existing index, concurrently when supported @@ -168,7 +168,7 @@ module Mastodon disable_statement_timeout end - remove_index(table_name, options.merge({ name: index_name })) + remove_index(table_name, **options.merge({ name: index_name })) end # Only available on Postgresql >= 9.2 @@ -472,7 +472,7 @@ module Mastodon col_opts[:limit] = old_col.limit end - add_column(table, new, new_type, col_opts) + add_column(table, new, new_type, **col_opts) # We set the default value _after_ adding the column so we don't end up # updating any existing data with the default value. This isn't @@ -510,10 +510,10 @@ module Mastodon new_pk_index_name = "index_#{table}_on_#{column}_cm" unless indexes_for(table, column).find{|i| i.name == old_pk_index_name} - add_concurrent_index(table, [temp_column], { + add_concurrent_index(table, [temp_column], unique: true, name: new_pk_index_name - }) + ) end end end @@ -763,7 +763,7 @@ module Mastodon options[:using] = index.using if index.using options[:where] = index.where if index.where - add_concurrent_index(table, new_columns, options) + add_concurrent_index(table, new_columns, **options) end end diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index 3f2a8f7c2..d341a8484 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -28,7 +28,7 @@ cache_namespace = namespace ? namespace + '_cache' : 'cache' REDIS_CACHE_PARAMS = { driver: :hiredis, - url: ENV['REDIS_URL'], + url: ENV['CACHE_REDIS_URL'], expires_in: 10.minutes, namespace: cache_namespace, }.freeze diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index 22a0acec8..c0491744b 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -53,7 +53,9 @@ module Mastodon index.specification.lock! end - ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1 + db_config = ActiveRecord::Base.configurations[Rails.env].dup + db_config['pool'] = options[:concurrency] + 1 + ActiveRecord::Base.establish_connection(db_config) pool = Concurrent::FixedThreadPool.new(options[:concurrency]) added = Concurrent::AtomicFixnum.new(0) diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index e25a34213..271f8b603 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -2,6 +2,10 @@ module Paperclip module AttachmentExtensions + def meta + instance_read(:meta) + end + # We overwrite this method to support delayed processing in # Sidekiq. Since we process the original file to reduce disk # usage, and we still want to generate thumbnails straight diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb index 9f3c8e8be..74aa1a0b2 100644 --- a/lib/paperclip/gif_transcoder.rb +++ b/lib/paperclip/gif_transcoder.rb @@ -100,7 +100,8 @@ end module Paperclip # This transcoder is only to be used for the MediaAttachment model - # to convert animated gifs to webm + # to convert animated GIFs to videos + class GifTranscoder < Paperclip::Processor def make return File.open(@file.path) unless needs_convert? diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index aab675a06..17fe4326f 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -31,21 +31,17 @@ module Paperclip private def extract_image_from_file! - ::Av.logger = Paperclip.logger - - cli = ::Av.cli dst = Tempfile.new([File.basename(@file.path, '.*'), '.png']) dst.binmode - cli.add_source(@file.path) - cli.add_destination(dst.path) - cli.add_output_param loglevel: 'fatal' - begin - cli.run - rescue Cocaine::ExitStatusError, ::Av::CommandError + command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) + command.run(source: @file.path, destination: dst.path, loglevel: 'fatal') + rescue Terrapin::ExitStatusError dst.close(true) return nil + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.' end dst diff --git a/lib/paperclip/schema_extensions.rb b/lib/paperclip/schema_extensions.rb new file mode 100644 index 000000000..8d065676a --- /dev/null +++ b/lib/paperclip/schema_extensions.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Monkey-patch various Paperclip methods for Ruby 3.0 compatibility + +module Paperclip + module Schema + module StatementsExtensions + def add_attachment(table_name, *attachment_names) + raise ArgumentError, 'Please specify attachment name in your add_attachment call in your migration.' if attachment_names.empty? + + options = attachment_names.extract_options! + + attachment_names.each do |attachment_name| + COLUMNS.each_pair do |column_name, column_type| + column_options = options.merge(options[column_name.to_sym] || {}) + add_column(table_name, "#{attachment_name}_#{column_name}", column_type, **column_options) + end + end + end + end + + module TableDefinitionExtensions + def attachment(*attachment_names) + options = attachment_names.extract_options! + attachment_names.each do |attachment_name| + COLUMNS.each_pair do |column_name, column_type| + column_options = options.merge(options[column_name.to_sym] || {}) + column("#{attachment_name}_#{column_name}", column_type, **column_options) + end + end + end + end + end +end + +Paperclip::Schema::Statements.prepend(Paperclip::Schema::StatementsExtensions) +Paperclip::Schema::TableDefinition.prepend(Paperclip::Schema::TableDefinitionExtensions) diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb new file mode 100644 index 000000000..e99704086 --- /dev/null +++ b/lib/paperclip/transcoder.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Paperclip + # This transcoder is only to be used for the MediaAttachment model + # to check when uploaded videos are actually gifv's + class Transcoder < Paperclip::Processor + def initialize(file, options = {}, attachment = nil) + super + + @current_format = File.extname(@file.path) + @basename = File.basename(@file.path, @current_format) + @format = options[:format] + @time = options[:time] || 3 + @passthrough_options = options[:passthrough_options] + @convert_options = options[:convert_options].dup + end + + def make + metadata = VideoMetadataExtractor.new(@file.path) + + unless metadata.valid? + log("Unsupported file #{@file.path}") + return File.open(@file.path) + end + + update_attachment_type(metadata) + update_options_from_metadata(metadata) + + destination = Tempfile.new([@basename, @format ? ".#{@format}" : '']) + destination.binmode + + @output_options = @convert_options[:output]&.dup || {} + @input_options = @convert_options[:input]&.dup || {} + + case @format.to_s + when /jpg$/, /jpeg$/, /png$/, /gif$/ + @input_options['ss'] = @time + + @output_options['f'] = 'image2' + @output_options['vframes'] = 1 + when 'mp4' + @output_options['acodec'] = 'aac' + @output_options['strict'] = 'experimental' + end + + command_arguments, interpolations = prepare_command(destination) + + begin + command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger) + command.run(interpolations) + rescue Terrapin::ExitStatusError => e + raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}" + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.' + end + + destination + end + + private + + def prepare_command(destination) + command_arguments = ['-nostdin'] + interpolations = {} + interpolation_keys = 0 + + @input_options.each_pair do |key, value| + interpolation_key = interpolation_keys + command_arguments << "-#{key} :#{interpolation_key}" + interpolations[interpolation_key] = value + interpolation_keys += 1 + end + + command_arguments << '-i :source' + interpolations[:source] = @file.path + + @output_options.each_pair do |key, value| + interpolation_key = interpolation_keys + command_arguments << "-#{key} :#{interpolation_key}" + interpolations[interpolation_key] = value + interpolation_keys += 1 + end + + command_arguments << '-y :destination' + interpolations[:destination] = destination.path + + [command_arguments, interpolations] + end + + def update_options_from_metadata(metadata) + return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + + @format = @passthrough_options[:options][:format] || @format + @time = @passthrough_options[:options][:time] || @time + @convert_options = @passthrough_options[:options][:convert_options].dup + end + + def update_attachment_type(metadata) + @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec + end + end +end diff --git a/lib/paperclip/transcoder_extensions.rb b/lib/paperclip/transcoder_extensions.rb deleted file mode 100644 index c0b2447f3..000000000 --- a/lib/paperclip/transcoder_extensions.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - module TranscoderExtensions - # Prevent the transcoder from modifying our meta hash - def initialize(file, options = {}, attachment = nil) - meta_value = attachment&.instance_read(:meta) - super - attachment&.instance_write(:meta, meta_value) - end - end -end - -Paperclip::Transcoder.prepend(Paperclip::TranscoderExtensions) diff --git a/lib/paperclip/validation_extensions.rb b/lib/paperclip/validation_extensions.rb new file mode 100644 index 000000000..0df0434f6 --- /dev/null +++ b/lib/paperclip/validation_extensions.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility + +module Paperclip + module Validators + module AttachmentSizeValidatorExtensions + def validate_each(record, attr_name, _value) + base_attr_name = attr_name + attr_name = "#{attr_name}_file_size".to_sym + value = record.send(:read_attribute_for_validation, attr_name) + + if value.present? + options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value| + option_value = option_value.call(record) if option_value.is_a?(Proc) + option_value = extract_option_value(option, option_value) + + next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value) + + error_message_key = options[:in] ? :in_between : option + [attr_name, base_attr_name].each do |error_attr_name| + record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge( + min: min_value_in_human_size(record), + max: max_value_in_human_size(record), + count: human_size(option_value) + )) + end + end + end + end + end + + module AttachmentContentTypeValidatorExtensions + def mark_invalid(record, attribute, types) + record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') }) + end + end + + module AttachmentPresenceValidatorExtensions + def validate_each(record, attribute, _value) + if record.send("#{attribute}_file_name").blank? + record.errors.add(attribute, :blank, **options) + end + end + end + + module AttachmentFileNameValidatorExtensions + def mark_invalid(record, attribute, patterns) + record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') }) + end + end + end +end + +Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions) +Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions) +Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions) +Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions) diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb deleted file mode 100644 index 4d9544231..000000000 --- a/lib/paperclip/video_transcoder.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - # This transcoder is only to be used for the MediaAttachment model - # to check when uploaded videos are actually gifv's - class VideoTranscoder < Paperclip::Processor - def make - movie = FFMPEG::Movie.new(@file.path) - - attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec - - Paperclip::Transcoder.make(file, actual_options(movie), attachment) - end - - private - - def actual_options(movie) - opts = options[:passthrough_options] - if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace) - opts[:options] - else - options - end - end - end -end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 7e6c1c8fc..b2a0d61de 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -22,7 +22,7 @@ namespace :db do unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate']) warn <<~WARNING Your database collation is susceptible to index corruption. - (This warning does not indicate that index corruption has occured and can be ignored) + (This warning does not indicate that index corruption has occurred and can be ignored) (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/) WARNING end diff --git a/lib/terrapin/multi_pipe_extensions.rb b/lib/terrapin/multi_pipe_extensions.rb new file mode 100644 index 000000000..209f4ad6c --- /dev/null +++ b/lib/terrapin/multi_pipe_extensions.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: false + +require 'fcntl' + +module Terrapin + module MultiPipeExtensions + def initialize + @stdout_in, @stdout_out = IO.pipe + @stderr_in, @stderr_out = IO.pipe + + clear_nonblocking_flags! + end + + def pipe_options + # Add some flags to explicitly close the other end of the pipes + { out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close } + end + + def read + # While we are patching Terrapin, fix child process potentially getting stuck on writing + # to stderr. + + @stdout_output = +'' + @stderr_output = +'' + + fds_to_read = [@stdout_in, @stderr_in] + until fds_to_read.empty? + rs, = IO.select(fds_to_read) + + read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in) + read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in) + end + end + + private + + # @param [IO] io IO Stream to read until there is nothing to read + # @param [String] result Mutable string to which read values will be appended to + # @param [Array<IO>] fds_to_read Mutable array from which `io` should be removed on EOF + def read_nonblocking!(io, result, fds_to_read) + while (partial_result = io.read_nonblock(8192)) + result << partial_result + end + rescue IO::WaitReadable + # Do nothing + rescue EOFError + fds_to_read.delete(io) + end + + def clear_nonblocking_flags! + # Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as + # needed when calling fork/exec-related syscalls, but posix-spawn does + # not currently do that, so we need to do it manually for the time being + # so that the child process do not error out when the buffers are full. + stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL) + @stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK + + stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL) + @stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK + rescue NameError, NotImplementedError, Errno::EINVAL + # Probably on windows, where pipes are blocking by default + end + end +end + +Terrapin::CommandLine::MultiPipe.prepend(Terrapin::MultiPipeExtensions) |