From 7aaf2b44ec698fd4f20b927fcac7edc0394a2647 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 30 Jun 2020 23:58:02 +0200 Subject: Fix remote files not using Content-Type header, streaming (#14184) --- lib/paperclip/image_extractor.rb | 45 ++++++++++-------- .../media_type_spoof_detector_extensions.rb | 27 +++++++++++ lib/paperclip/response_with_limit_adapter.rb | 55 ++++++++++++++++++++++ 3 files changed, 107 insertions(+), 20 deletions(-) create mode 100644 lib/paperclip/media_type_spoof_detector_extensions.rb create mode 100644 lib/paperclip/response_with_limit_adapter.rb (limited to 'lib') diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index 114852e8b..f5a54d1a5 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -4,28 +4,10 @@ require 'mime/types/columnar' module Paperclip class ImageExtractor < Paperclip::Processor - IMAGE_EXTRACTION_OPTIONS = { - convert_options: { - output: { - 'loglevel' => 'fatal', - vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease', - }.freeze, - }.freeze, - format: 'png', - time: -1, - file_geometry_parser: FastGeometryParser, - }.freeze - def make return @file unless options[:style] == :original - image = begin - begin - Paperclip::Transcoder.make(file, IMAGE_EXTRACTION_OPTIONS.dup, attachment) - rescue Paperclip::Error, ::Av::CommandError - nil - end - end + image = extract_image_from_file! unless image.nil? begin @@ -36,7 +18,7 @@ module Paperclip # to make sure it's cleaned up begin - FileUtils.rm(image) + image.close(true) rescue Errno::ENOENT nil end @@ -45,5 +27,28 @@ module Paperclip @file end + + private + + def extract_image_from_file! + ::Av.logger = Paperclip.logger + + cli = ::Av.cli + dst = Tempfile.new([File.basename(@file.path, '.*'), '.png']) + dst.binmode + + cli.add_source(@file.path) + cli.add_destination(dst.path) + cli.add_output_param loglevel: 'fatal' + + begin + cli.run + rescue Cocaine::ExitStatusError + dst.close(true) + return nil + end + + dst + end end end diff --git a/lib/paperclip/media_type_spoof_detector_extensions.rb b/lib/paperclip/media_type_spoof_detector_extensions.rb new file mode 100644 index 000000000..9c0557356 --- /dev/null +++ b/lib/paperclip/media_type_spoof_detector_extensions.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Paperclip + module MediaTypeSpoofDetectorExtensions + def calculated_content_type + @calculated_content_type ||= type_from_mime_magic || type_from_file_command + end + + def type_from_mime_magic + @type_from_mime_magic ||= begin + begin + File.open(@file.path) do |file| + MimeMagic.by_magic(file)&.type + end + rescue Errno::ENOENT + '' + end + end + end + + def type_from_file_command + @type_from_file_command ||= FileCommandContentTypeDetector.new(@file.path).detect + end + end +end + +Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions) diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb new file mode 100644 index 000000000..7d897b8d6 --- /dev/null +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Paperclip + class ResponseWithLimitAdapter < AbstractAdapter + def self.register + Paperclip.io_adapters.register self do |target| + target.is_a?(ResponseWithLimit) + end + end + + def initialize(target, options = {}) + super + cache_current_values + end + + private + + def cache_current_values + @original_filename = filename_from_content_disposition || filename_from_path || 'data' + @size = @target.response.content_length + @tempfile = copy_to_tempfile(@target) + @content_type = @target.response.mime_type || ContentTypeDetector.new(@tempfile.path).detect + end + + def copy_to_tempfile(source) + bytes_read = 0 + + source.response.body.each do |chunk| + bytes_read += chunk.bytesize + + destination.write(chunk) + chunk.clear + + raise Mastodon::LengthValidationError if bytes_read > source.limit + end + + destination.rewind + destination + rescue Mastodon::LengthValidationError + destination.close(true) + raise + ensure + source.response.connection.close + end + + def filename_from_content_disposition + disposition = @target.response.headers['content-disposition'] + disposition&.match(/filename="([^"]*)"/)&.captures&.first + end + + def filename_from_path + @target.response.uri.path.split('/').last + end + end +end -- cgit From 6d23d40420e4548778f3ca4ed9e8cb16e0eb0073 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 1 Jul 2020 19:05:21 +0200 Subject: Change Redis#exists calls to Redis#exists? to avoid deprecation warning (#14191) --- .gitignore | 28 +++++++++++----------- Gemfile.lock | 7 ++---- app/lib/activitypub/activity.rb | 2 +- app/lib/activitypub/activity/move.rb | 2 +- app/lib/feed_manager.rb | 2 +- app/models/account_conversation.rb | 2 +- app/models/encrypted_message.rb | 5 +--- app/models/home_feed.rb | 2 +- .../publish_announcement_reaction_worker.rb | 2 +- .../publish_scheduled_announcement_worker.rb | 2 +- app/workers/unpublish_announcement_worker.rb | 2 +- config/application.rb | 1 + config/initializers/redis.rb | 2 -- lib/redis/namespace_extensions.rb | 12 ++++++++++ 14 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 lib/redis/namespace_extensions.rb (limited to 'lib') diff --git a/.gitignore b/.gitignore index 9f6c4b413..4545270b3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,36 +17,36 @@ /log/* !/log/.keep /tmp -coverage -public/system -public/assets -public/packs -public/packs-test +/coverage +/public/system +/public/assets +/public/packs +/public/packs-test .env .env.production .env.development -node_modules/ -build/ +/node_modules/ +/build/ # Ignore Vagrant files .vagrant/ # Ignore Capistrano customizations -config/deploy/* +/config/deploy/* # Ignore IDE files .vscode/ .idea/ # Ignore postgres + redis + elasticsearch volume optionally created by docker-compose -postgres -redis -elasticsearch +/postgres +/redis +/elasticsearch # ignore Helm lockfile, dependency charts, and local values file -chart/Chart.lock -chart/charts/*.tgz -chart/values.yaml +/chart/Chart.lock +/chart/charts/*.tgz +/chart/values.yaml # Ignore Apple files .DS_Store diff --git a/Gemfile.lock b/Gemfile.lock index 96572bde5..fcea81002 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -444,8 +444,6 @@ GEM rack (>= 1.0, < 3) rack-cors (1.1.1) rack (>= 2.0.0) - rack-protection (2.0.8.1) - rack rack-proxy (0.6.5) rack rack-test (1.1.0) @@ -570,11 +568,10 @@ GEM nokogiri (>= 1.8.0) nokogumbo (~> 2.0) semantic_range (2.3.0) - sidekiq (6.0.7) + sidekiq (6.1.0) connection_pool (>= 2.2.2) rack (~> 2.0) - rack-protection (>= 2.0.0) - redis (>= 4.1.0) + redis (>= 4.2.0) sidekiq-bulk (0.2.0) sidekiq sidekiq-scheduler (3.0.1) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index ee35e1e8d..58cec7ac4 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -132,7 +132,7 @@ class ActivityPub::Activity end def delete_arrived_first?(uri) - redis.exists("delete_upon_arrival:#{@account.id}:#{uri}") + redis.exists?("delete_upon_arrival:#{@account.id}:#{uri}") end def delete_later!(uri) diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 12bb82d25..2103f503f 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -33,7 +33,7 @@ class ActivityPub::Activity::Move < ActivityPub::Activity end def processed? - redis.exists("move_in_progress:#{@account.id}") + redis.exists?("move_in_progress:#{@account.id}") end def mark_as_processing! diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index efb4f6e2c..53ff31f5e 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -169,7 +169,7 @@ class FeedManager private def push_update_required?(timeline_id) - redis.exists("subscribed:#{timeline_id}") + redis.exists?("subscribed:#{timeline_id}") end def blocks_or_mutes?(receiver_id, account_ids, context) diff --git a/app/models/account_conversation.rb b/app/models/account_conversation.rb index 0c03747e2..b43816588 100644 --- a/app/models/account_conversation.rb +++ b/app/models/account_conversation.rb @@ -108,7 +108,7 @@ class AccountConversation < ApplicationRecord end def subscribed_to_timeline? - Redis.current.exists("subscribed:#{streaming_channel}") + Redis.current.exists?("subscribed:#{streaming_channel}") end def streaming_channel diff --git a/app/models/encrypted_message.rb b/app/models/encrypted_message.rb index 5e0aba434..aa4182b4e 100644 --- a/app/models/encrypted_message.rb +++ b/app/models/encrypted_message.rb @@ -32,16 +32,13 @@ class EncryptedMessage < ApplicationRecord private def push_to_streaming_api - Rails.logger.info(streaming_channel) - Rails.logger.info(subscribed_to_timeline?) - return if destroyed? || !subscribed_to_timeline? PushEncryptedMessageWorker.perform_async(id) end def subscribed_to_timeline? - Redis.current.exists("subscribed:#{streaming_channel}") + Redis.current.exists?("subscribed:#{streaming_channel}") end def streaming_channel diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb index 1fd506138..0fe9dae46 100644 --- a/app/models/home_feed.rb +++ b/app/models/home_feed.rb @@ -8,6 +8,6 @@ class HomeFeed < Feed end def regenerating? - redis.exists("account:#{@id}:regeneration") + redis.exists?("account:#{@id}:regeneration") end end diff --git a/app/workers/publish_announcement_reaction_worker.rb b/app/workers/publish_announcement_reaction_worker.rb index 418dc7127..03da56550 100644 --- a/app/workers/publish_announcement_reaction_worker.rb +++ b/app/workers/publish_announcement_reaction_worker.rb @@ -14,7 +14,7 @@ class PublishAnnouncementReactionWorker payload = Oj.dump(event: :'announcement.reaction', payload: payload) FeedManager.instance.with_active_accounts do |account| - redis.publish("timeline:#{account.id}", payload) if redis.exists("subscribed:timeline:#{account.id}") + redis.publish("timeline:#{account.id}", payload) if redis.exists?("subscribed:timeline:#{account.id}") end rescue ActiveRecord::RecordNotFound true diff --git a/app/workers/publish_scheduled_announcement_worker.rb b/app/workers/publish_scheduled_announcement_worker.rb index 1392efed0..c23eae6af 100644 --- a/app/workers/publish_scheduled_announcement_worker.rb +++ b/app/workers/publish_scheduled_announcement_worker.rb @@ -15,7 +15,7 @@ class PublishScheduledAnnouncementWorker payload = Oj.dump(event: :announcement, payload: payload) FeedManager.instance.with_active_accounts do |account| - redis.publish("timeline:#{account.id}", payload) if redis.exists("subscribed:timeline:#{account.id}") + redis.publish("timeline:#{account.id}", payload) if redis.exists?("subscribed:timeline:#{account.id}") end end diff --git a/app/workers/unpublish_announcement_worker.rb b/app/workers/unpublish_announcement_worker.rb index e99d70cf8..e58c07554 100644 --- a/app/workers/unpublish_announcement_worker.rb +++ b/app/workers/unpublish_announcement_worker.rb @@ -8,7 +8,7 @@ class UnpublishAnnouncementWorker payload = Oj.dump(event: :'announcement.delete', payload: announcement_id.to_s) FeedManager.instance.with_active_accounts do |account| - redis.publish("timeline:#{account.id}", payload) if redis.exists("subscribed:timeline:#{account.id}") + redis.publish("timeline:#{account.id}", payload) if redis.exists?("subscribed:timeline:#{account.id}") end end end diff --git a/config/application.rb b/config/application.rb index a1da9d61f..a3c37b042 100644 --- a/config/application.rb +++ b/config/application.rb @@ -7,6 +7,7 @@ require 'rails/all' Bundler.require(*Rails.groups) require_relative '../app/lib/exceptions' +require_relative '../lib/redis/namespace_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/media_type_spoof_detector_extensions' diff --git a/config/initializers/redis.rb b/config/initializers/redis.rb index 510194044..7573fc9f7 100644 --- a/config/initializers/redis.rb +++ b/config/initializers/redis.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -Redis.exists_returns_integer = false - redis_connection = Redis.new( url: ENV['REDIS_URL'], driver: :hiredis diff --git a/lib/redis/namespace_extensions.rb b/lib/redis/namespace_extensions.rb new file mode 100644 index 000000000..310a4f465 --- /dev/null +++ b/lib/redis/namespace_extensions.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class Redis + module NamespaceExtensions + def exists?(*args, &block) + call_with_namespace('exists?', *args, &block) + end + end +end + +Redis::Namespace::COMMANDS['exists?'] = [:first] +Redis::Namespace.prepend(Redis::NamespaceExtensions) -- cgit