From ad61265268f13d9b2a04e2e176724d8a7376f85a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 12 Apr 2021 03:35:58 +0200 Subject: Remove dependency on pluck_each gem (#16012) --- lib/active_record/batches.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lib/active_record/batches.rb (limited to 'lib') diff --git a/lib/active_record/batches.rb b/lib/active_record/batches.rb new file mode 100644 index 000000000..55d29e52e --- /dev/null +++ b/lib/active_record/batches.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module ActiveRecord + module Batches + def pluck_each(*column_names) + relation = self + + options = column_names.extract_options! + + flatten = column_names.size == 1 + batch_limit = options[:batch_limit] || 1_000 + order = options[:order] || :asc + + column_names.unshift(primary_key) + + relation = relation.reorder(batch_order(order)).limit(batch_limit) + relation.skip_query_cache! + + batch_relation = relation + + loop do + batch = batch_relation.pluck(*column_names) + + break if batch.empty? + + primary_key_offset = batch.last[0] + + batch.each do |record| + if flatten + yield record[1] + else + yield record[1..-1] + end + end + + break if batch.size < batch_limit + + batch_relation = relation.where( + predicate_builder[primary_key, primary_key_offset, order == :desc ? :lt : :gt] + ) + end + end + end +end -- cgit From e78d06eecfb21de6aedf39fd7c63d9aa68f7033c Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 13 Apr 2021 23:43:51 +0200 Subject: Add border to ๐Ÿšฒ emoji (#16035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/javascript/mastodon/features/emoji/emoji.js | 2 +- lib/tasks/emojis.rake | 2 +- public/emoji/1f6b2_border.svg | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 public/emoji/1f6b2_border.svg (limited to 'lib') diff --git a/app/javascript/mastodon/features/emoji/emoji.js b/app/javascript/mastodon/features/emoji/emoji.js index 3de79ac9b..fb1a3804c 100644 --- a/app/javascript/mastodon/features/emoji/emoji.js +++ b/app/javascript/mastodon/features/emoji/emoji.js @@ -11,7 +11,7 @@ const emojiFilenames = (emojis) => { }; // Emoji requiring extra borders depending on theme -const darkEmoji = emojiFilenames(['๐ŸŽฑ', '๐Ÿœ', 'โšซ', '๐Ÿ–ค', 'โฌ›', 'โ—ผ๏ธ', 'โ—พ', 'โ—ผ๏ธ', 'โœ’๏ธ', 'โ–ช๏ธ', '๐Ÿ’ฃ', '๐ŸŽณ', '๐Ÿ“ท', '๐Ÿ“ธ', 'โ™ฃ๏ธ', '๐Ÿ•ถ๏ธ', 'โœด๏ธ', '๐Ÿ”Œ', '๐Ÿ’‚โ€โ™€๏ธ', '๐Ÿ“ฝ๏ธ', '๐Ÿณ', '๐Ÿฆ', '๐Ÿ’‚', '๐Ÿ”ช', '๐Ÿ•ณ๏ธ', '๐Ÿ•น๏ธ', '๐Ÿ•‹', '๐Ÿ–Š๏ธ', '๐Ÿ–‹๏ธ', '๐Ÿ’‚โ€โ™‚๏ธ', '๐ŸŽค', '๐ŸŽ“', '๐ŸŽฅ', '๐ŸŽผ', 'โ™ ๏ธ', '๐ŸŽฉ', '๐Ÿฆƒ', '๐Ÿ“ผ', '๐Ÿ“น', '๐ŸŽฎ', '๐Ÿƒ', '๐Ÿด', '๐Ÿž', '๐Ÿ•บ', '๐Ÿ“ฑ', '๐Ÿ“ฒ']); +const darkEmoji = emojiFilenames(['๐ŸŽฑ', '๐Ÿœ', 'โšซ', '๐Ÿ–ค', 'โฌ›', 'โ—ผ๏ธ', 'โ—พ', 'โ—ผ๏ธ', 'โœ’๏ธ', 'โ–ช๏ธ', '๐Ÿ’ฃ', '๐ŸŽณ', '๐Ÿ“ท', '๐Ÿ“ธ', 'โ™ฃ๏ธ', '๐Ÿ•ถ๏ธ', 'โœด๏ธ', '๐Ÿ”Œ', '๐Ÿ’‚โ€โ™€๏ธ', '๐Ÿ“ฝ๏ธ', '๐Ÿณ', '๐Ÿฆ', '๐Ÿ’‚', '๐Ÿ”ช', '๐Ÿ•ณ๏ธ', '๐Ÿ•น๏ธ', '๐Ÿ•‹', '๐Ÿ–Š๏ธ', '๐Ÿ–‹๏ธ', '๐Ÿ’‚โ€โ™‚๏ธ', '๐ŸŽค', '๐ŸŽ“', '๐ŸŽฅ', '๐ŸŽผ', 'โ™ ๏ธ', '๐ŸŽฉ', '๐Ÿฆƒ', '๐Ÿ“ผ', '๐Ÿ“น', '๐ŸŽฎ', '๐Ÿƒ', '๐Ÿด', '๐Ÿž', '๐Ÿ•บ', '๐Ÿ“ฑ', '๐Ÿ“ฒ', '๐Ÿšฒ']); const lightEmoji = emojiFilenames(['๐Ÿ‘ฝ', 'โšพ', '๐Ÿ”', 'โ˜๏ธ', '๐Ÿ’จ', '๐Ÿ•Š๏ธ', '๐Ÿ‘€', '๐Ÿฅ', '๐Ÿ‘ป', '๐Ÿ', 'โ•', 'โ”', 'โ›ธ๏ธ', '๐ŸŒฉ๏ธ', '๐Ÿ”Š', '๐Ÿ”‡', '๐Ÿ“ƒ', '๐ŸŒง๏ธ', '๐Ÿ', '๐Ÿš', '๐Ÿ™', '๐Ÿ“', '๐Ÿ‘', '๐Ÿ’€', 'โ˜ ๏ธ', '๐ŸŒจ๏ธ', '๐Ÿ”‰', '๐Ÿ”ˆ', '๐Ÿ’ฌ', '๐Ÿ’ญ', '๐Ÿ', '๐Ÿณ๏ธ', 'โšช', 'โฌœ', 'โ—ฝ', 'โ—ป๏ธ', 'โ–ซ๏ธ']); const emojiFilename = (filename) => { diff --git a/lib/tasks/emojis.rake b/lib/tasks/emojis.rake index c8655cc47..a373e7652 100644 --- a/lib/tasks/emojis.rake +++ b/lib/tasks/emojis.rake @@ -91,7 +91,7 @@ namespace :emojis do desc 'Generate emoji variants with white borders' task :generate_borders do src = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json') - emojis = '๐ŸŽฑ๐Ÿœโšซ๐Ÿ–คโฌ›โ—ผ๏ธโ—พโ—ผ๏ธโœ’๏ธโ–ช๏ธ๐Ÿ’ฃ๐ŸŽณ๐Ÿ“ท๐Ÿ“ธโ™ฃ๏ธ๐Ÿ•ถ๏ธโœด๏ธ๐Ÿ”Œ๐Ÿ’‚โ€โ™€๏ธ๐Ÿ“ฝ๏ธ๐Ÿณ๐Ÿฆ๐Ÿ’‚๐Ÿ”ช๐Ÿ•ณ๏ธ๐Ÿ•น๏ธ๐Ÿ•‹๐Ÿ–Š๏ธ๐Ÿ–‹๏ธ๐Ÿ’‚โ€โ™‚๏ธ๐ŸŽค๐ŸŽ“๐ŸŽฅ๐ŸŽผโ™ ๏ธ๐ŸŽฉ๐Ÿฆƒ๐Ÿ“ผ๐Ÿ“น๐ŸŽฎ๐Ÿƒ๐Ÿด๐Ÿž๐Ÿ•บ๐Ÿ“ฑ๐Ÿ“ฒ๐Ÿ‘ฝโšพ๐Ÿ”โ˜๏ธ๐Ÿ’จ๐Ÿ•Š๏ธ๐Ÿ‘€๐Ÿฅ๐Ÿ‘ป๐Ÿโ•โ”โ›ธ๏ธ๐ŸŒฉ๏ธ๐Ÿ”Š๐Ÿ”‡๐Ÿ“ƒ๐ŸŒง๏ธ๐Ÿ๐Ÿš๐Ÿ™๐Ÿ“๐Ÿ‘๐Ÿ’€โ˜ ๏ธ๐ŸŒจ๏ธ๐Ÿ”‰๐Ÿ”ˆ๐Ÿ’ฌ๐Ÿ’ญ๐Ÿ๐Ÿณ๏ธโšชโฌœโ—ฝโ—ป๏ธโ–ซ๏ธ' + emojis = '๐ŸŽฑ๐Ÿœโšซ๐Ÿ–คโฌ›โ—ผ๏ธโ—พโ—ผ๏ธโœ’๏ธโ–ช๏ธ๐Ÿ’ฃ๐ŸŽณ๐Ÿ“ท๐Ÿ“ธโ™ฃ๏ธ๐Ÿ•ถ๏ธโœด๏ธ๐Ÿ”Œ๐Ÿ’‚โ€โ™€๏ธ๐Ÿ“ฝ๏ธ๐Ÿณ๐Ÿฆ๐Ÿ’‚๐Ÿ”ช๐Ÿ•ณ๏ธ๐Ÿ•น๏ธ๐Ÿ•‹๐Ÿ–Š๏ธ๐Ÿ–‹๏ธ๐Ÿ’‚โ€โ™‚๏ธ๐ŸŽค๐ŸŽ“๐ŸŽฅ๐ŸŽผโ™ ๏ธ๐ŸŽฉ๐Ÿฆƒ๐Ÿ“ผ๐Ÿ“น๐ŸŽฎ๐Ÿƒ๐Ÿด๐Ÿž๐Ÿ•บ๐Ÿ“ฑ๐Ÿ“ฒ๐Ÿšฒ๐Ÿ‘ฝโšพ๐Ÿ”โ˜๏ธ๐Ÿ’จ๐Ÿ•Š๏ธ๐Ÿ‘€๐Ÿฅ๐Ÿ‘ป๐Ÿโ•โ”โ›ธ๏ธ๐ŸŒฉ๏ธ๐Ÿ”Š๐Ÿ”‡๐Ÿ“ƒ๐ŸŒง๏ธ๐Ÿ๐Ÿš๐Ÿ™๐Ÿ“๐Ÿ‘๐Ÿ’€โ˜ ๏ธ๐ŸŒจ๏ธ๐Ÿ”‰๐Ÿ”ˆ๐Ÿ’ฌ๐Ÿ’ญ๐Ÿ๐Ÿณ๏ธโšชโฌœโ—ฝโ—ป๏ธโ–ซ๏ธ' map = Oj.load(File.read(src)) diff --git a/public/emoji/1f6b2_border.svg b/public/emoji/1f6b2_border.svg new file mode 100644 index 000000000..0219841a1 --- /dev/null +++ b/public/emoji/1f6b2_border.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + -- cgit From 0bc909687af6d5176318fc82db8b497dfff040e1 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Tue, 27 Apr 2021 23:25:24 +0900 Subject: Fix typo in db.rake (#16126) occured -> occurred --- lib/tasks/db.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 7e6c1c8fc..b2a0d61de 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -22,7 +22,7 @@ namespace :db do unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate']) warn <<~WARNING Your database collation is susceptible to index corruption. - (This warning does not indicate that index corruption has occured and can be ignored) + (This warning does not indicate that index corruption has occurred and can be ignored) (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/) WARNING end -- cgit From 422df9d670dea235089e5d0732f50bc45bd3d673 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 29 Apr 2021 22:43:49 +0900 Subject: Fix cache redis not being used (#16131) --- lib/mastodon/redis_config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index 3f2a8f7c2..d341a8484 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -28,7 +28,7 @@ cache_namespace = namespace ? namespace + '_cache' : 'cache' REDIS_CACHE_PARAMS = { driver: :hiredis, - url: ENV['REDIS_URL'], + url: ENV['CACHE_REDIS_URL'], expires_in: 10.minutes, namespace: cache_namespace, }.freeze -- cgit From 25345c90ffb74a8231eae4a130c98b6eaad7ce36 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Wed, 5 May 2021 11:41:10 +0900 Subject: Fix how to change connection pool for rails 6 (#16158) * Fix how to change connection pool for rails 6 * Fix to match the code elsewhere --- lib/mastodon/search_cli.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index 22a0acec8..c0491744b 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -53,7 +53,9 @@ module Mastodon index.specification.lock! end - ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1 + db_config = ActiveRecord::Base.configurations[Rails.env].dup + db_config['pool'] = options[:concurrency] + 1 + ActiveRecord::Base.establish_connection(db_config) pool = Concurrent::FixedThreadPool.new(options[:concurrency]) added = Concurrent::AtomicFixnum.new(0) -- cgit From 036556d3509fac5fa487a0d5ff3cf95767e8d84f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 19:44:01 +0200 Subject: Fix media processing getting stuck on too much stdin/stderr (#16136) * Fix media processing getting stuck on too much stdin/stderr See thoughtbot/terrapin#5 * Remove dependency on paperclip-av-transcoder gem * Remove dependency on streamio-ffmpeg gem * Disable stdin on ffmpeg process --- Gemfile | 2 - Gemfile.lock | 11 ---- app/lib/video_metadata_extractor.rb | 54 +++++++++++++++++ app/models/media_attachment.rb | 4 +- config/application.rb | 4 +- lib/paperclip/attachment_extensions.rb | 4 ++ lib/paperclip/gif_transcoder.rb | 3 +- lib/paperclip/image_extractor.rb | 14 ++--- lib/paperclip/transcoder.rb | 102 +++++++++++++++++++++++++++++++++ lib/paperclip/transcoder_extensions.rb | 14 ----- lib/paperclip/video_transcoder.rb | 26 --------- lib/terrapin/multi_pipe_extensions.rb | 63 ++++++++++++++++++++ 12 files changed, 234 insertions(+), 67 deletions(-) create mode 100644 app/lib/video_metadata_extractor.rb create mode 100644 lib/paperclip/transcoder.rb delete mode 100644 lib/paperclip/transcoder_extensions.rb delete mode 100644 lib/paperclip/video_transcoder.rb create mode 100644 lib/terrapin/multi_pipe_extensions.rb (limited to 'lib') diff --git a/Gemfile b/Gemfile index fb80e24d5..6ca0a81de 100644 --- a/Gemfile +++ b/Gemfile @@ -21,8 +21,6 @@ gem 'aws-sdk-s3', '~> 1.94', require: false gem 'fog-core', '<= 2.1.0' gem 'fog-openstack', '~> 0.3', require: false gem 'paperclip', '~> 6.0' -gem 'paperclip-av-transcoder', '~> 0.6' -gem 'streamio-ffmpeg', '~> 3.0' gem 'blurhash', '~> 0.1' gem 'active_model_serializers', '~> 0.10' diff --git a/Gemfile.lock b/Gemfile.lock index 3c36e07bc..b1ae4fd22 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,8 +77,6 @@ GEM ast (2.4.2) attr_encrypted (3.1.0) encryptor (~> 3.0.0) - av (0.9.0) - cocaine (~> 0.5.3) awrence (1.1.1) aws-eventstream (1.1.1) aws-partitions (1.449.0) @@ -156,8 +154,6 @@ GEM cld3 (3.4.2) ffi (>= 1.1.0, < 1.16.0) climate_control (0.2.0) - cocaine (0.5.8) - climate_control (>= 0.0.3, < 1.0) coderay (1.1.3) color_diff (0.1) concurrent-ruby (1.1.8) @@ -402,9 +398,6 @@ GEM mime-types mimemagic (~> 0.3.0) terrapin (~> 0.6.0) - paperclip-av-transcoder (0.6.4) - av (~> 0.9.0) - paperclip (>= 2.5.2) parallel (1.20.1) parallel_tests (3.7.0) parallel @@ -605,8 +598,6 @@ GEM stackprof (0.2.16) statsd-ruby (1.5.0) stoplight (2.2.1) - streamio-ffmpeg (3.0.2) - multi_json (~> 1.8) strong_migrations (0.7.6) activerecord (>= 5) temple (0.8.2) @@ -750,7 +741,6 @@ DEPENDENCIES omniauth-saml (~> 1.10) ox (~> 2.14) paperclip (~> 6.0) - paperclip-av-transcoder (~> 0.6) parallel (~> 1.20) parallel_tests (~> 3.7) parslet @@ -795,7 +785,6 @@ DEPENDENCIES sprockets-rails (~> 3.2) stackprof stoplight (~> 2.2.1) - streamio-ffmpeg (~> 3.0) strong_migrations (~> 0.7) thor (~> 1.1) tty-prompt (~> 0.23) diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb new file mode 100644 index 000000000..03e40f923 --- /dev/null +++ b/app/lib/video_metadata_extractor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class VideoMetadataExtractor + attr_reader :duration, :bitrate, :video_codec, :audio_codec, + :colorspace, :width, :height, :frame_rate + + def initialize(path) + @path = path + @metadata = Oj.load(ffmpeg_command_output, mode: :strict, symbol_keys: true) + + parse_metadata + rescue Terrapin::ExitStatusError, Oj::ParseError + @invalid = true + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffprobe` command. Please install ffmpeg.' + end + + def valid? + !@invalid + end + + private + + def ffmpeg_command_output + command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') + command.run(path: @path, format: 'json', loglevel: 'fatal') + end + + def parse_metadata + if @metadata.key?(:format) + @duration = @metadata[:format][:duration].to_f + @bitrate = @metadata[:format][:bit_rate].to_i + end + + if @metadata.key?(:streams) + video_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'video' } + audio_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'audio' } + + if (video_stream = video_streams.first) + @video_codec = video_stream[:codec_name] + @colorspace = video_stream[:pix_fmt] + @width = video_stream[:width] + @height = video_stream[:height] + @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) + end + + if (audio_stream = audio_streams.first) + @audio_codec = audio_stream[:codec_name] + end + end + + @invalid = true if @metadata.key?(:error) + end +end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 5cf4d8127..3515f6895 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -287,7 +287,7 @@ class MediaAttachment < ApplicationRecord if instance.file_content_type == 'image/gif' [:gif_transcoder, :blurhash_transcoder] elsif VIDEO_MIME_TYPES.include?(instance.file_content_type) - [:video_transcoder, :blurhash_transcoder, :type_corrector] + [:transcoder, :blurhash_transcoder, :type_corrector] elsif AUDIO_MIME_TYPES.include?(instance.file_content_type) [:image_extractor, :transcoder, :type_corrector] else @@ -388,7 +388,7 @@ class MediaAttachment < ApplicationRecord # paths but ultimately the same file, so it makes sense to memoize the # result while disregarding the path def ffmpeg_data(path = nil) - @ffmpeg_data ||= FFMPEG::Movie.new(path) + @ffmpeg_data ||= VideoMetadataExtractor.new(path) end def enqueue_processing diff --git a/config/application.rb b/config/application.rb index 9aa1594ce..37a996224 100644 --- a/config/application.rb +++ b/config/application.rb @@ -13,12 +13,12 @@ require_relative '../lib/redis/namespace_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/media_type_spoof_detector_extensions' -require_relative '../lib/paperclip/transcoder_extensions' require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/gif_transcoder' -require_relative '../lib/paperclip/video_transcoder' +require_relative '../lib/paperclip/transcoder' require_relative '../lib/paperclip/type_corrector' require_relative '../lib/paperclip/response_with_limit_adapter' +require_relative '../lib/terrapin/multi_pipe_extensions' require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' require_relative '../lib/devise/two_factor_ldap_authenticatable' diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index e25a34213..271f8b603 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -2,6 +2,10 @@ module Paperclip module AttachmentExtensions + def meta + instance_read(:meta) + end + # We overwrite this method to support delayed processing in # Sidekiq. Since we process the original file to reduce disk # usage, and we still want to generate thumbnails straight diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb index 9f3c8e8be..74aa1a0b2 100644 --- a/lib/paperclip/gif_transcoder.rb +++ b/lib/paperclip/gif_transcoder.rb @@ -100,7 +100,8 @@ end module Paperclip # This transcoder is only to be used for the MediaAttachment model - # to convert animated gifs to webm + # to convert animated GIFs to videos + class GifTranscoder < Paperclip::Processor def make return File.open(@file.path) unless needs_convert? diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index aab675a06..17fe4326f 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -31,21 +31,17 @@ module Paperclip private def extract_image_from_file! - ::Av.logger = Paperclip.logger - - cli = ::Av.cli dst = Tempfile.new([File.basename(@file.path, '.*'), '.png']) dst.binmode - cli.add_source(@file.path) - cli.add_destination(dst.path) - cli.add_output_param loglevel: 'fatal' - begin - cli.run - rescue Cocaine::ExitStatusError, ::Av::CommandError + command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) + command.run(source: @file.path, destination: dst.path, loglevel: 'fatal') + rescue Terrapin::ExitStatusError dst.close(true) return nil + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.' end dst diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb new file mode 100644 index 000000000..e99704086 --- /dev/null +++ b/lib/paperclip/transcoder.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Paperclip + # This transcoder is only to be used for the MediaAttachment model + # to check when uploaded videos are actually gifv's + class Transcoder < Paperclip::Processor + def initialize(file, options = {}, attachment = nil) + super + + @current_format = File.extname(@file.path) + @basename = File.basename(@file.path, @current_format) + @format = options[:format] + @time = options[:time] || 3 + @passthrough_options = options[:passthrough_options] + @convert_options = options[:convert_options].dup + end + + def make + metadata = VideoMetadataExtractor.new(@file.path) + + unless metadata.valid? + log("Unsupported file #{@file.path}") + return File.open(@file.path) + end + + update_attachment_type(metadata) + update_options_from_metadata(metadata) + + destination = Tempfile.new([@basename, @format ? ".#{@format}" : '']) + destination.binmode + + @output_options = @convert_options[:output]&.dup || {} + @input_options = @convert_options[:input]&.dup || {} + + case @format.to_s + when /jpg$/, /jpeg$/, /png$/, /gif$/ + @input_options['ss'] = @time + + @output_options['f'] = 'image2' + @output_options['vframes'] = 1 + when 'mp4' + @output_options['acodec'] = 'aac' + @output_options['strict'] = 'experimental' + end + + command_arguments, interpolations = prepare_command(destination) + + begin + command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger) + command.run(interpolations) + rescue Terrapin::ExitStatusError => e + raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}" + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.' + end + + destination + end + + private + + def prepare_command(destination) + command_arguments = ['-nostdin'] + interpolations = {} + interpolation_keys = 0 + + @input_options.each_pair do |key, value| + interpolation_key = interpolation_keys + command_arguments << "-#{key} :#{interpolation_key}" + interpolations[interpolation_key] = value + interpolation_keys += 1 + end + + command_arguments << '-i :source' + interpolations[:source] = @file.path + + @output_options.each_pair do |key, value| + interpolation_key = interpolation_keys + command_arguments << "-#{key} :#{interpolation_key}" + interpolations[interpolation_key] = value + interpolation_keys += 1 + end + + command_arguments << '-y :destination' + interpolations[:destination] = destination.path + + [command_arguments, interpolations] + end + + def update_options_from_metadata(metadata) + return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + + @format = @passthrough_options[:options][:format] || @format + @time = @passthrough_options[:options][:time] || @time + @convert_options = @passthrough_options[:options][:convert_options].dup + end + + def update_attachment_type(metadata) + @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec + end + end +end diff --git a/lib/paperclip/transcoder_extensions.rb b/lib/paperclip/transcoder_extensions.rb deleted file mode 100644 index c0b2447f3..000000000 --- a/lib/paperclip/transcoder_extensions.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - module TranscoderExtensions - # Prevent the transcoder from modifying our meta hash - def initialize(file, options = {}, attachment = nil) - meta_value = attachment&.instance_read(:meta) - super - attachment&.instance_write(:meta, meta_value) - end - end -end - -Paperclip::Transcoder.prepend(Paperclip::TranscoderExtensions) diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb deleted file mode 100644 index 4d9544231..000000000 --- a/lib/paperclip/video_transcoder.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - # This transcoder is only to be used for the MediaAttachment model - # to check when uploaded videos are actually gifv's - class VideoTranscoder < Paperclip::Processor - def make - movie = FFMPEG::Movie.new(@file.path) - - attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec - - Paperclip::Transcoder.make(file, actual_options(movie), attachment) - end - - private - - def actual_options(movie) - opts = options[:passthrough_options] - if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace) - opts[:options] - else - options - end - end - end -end diff --git a/lib/terrapin/multi_pipe_extensions.rb b/lib/terrapin/multi_pipe_extensions.rb new file mode 100644 index 000000000..51d7de37c --- /dev/null +++ b/lib/terrapin/multi_pipe_extensions.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: false +# Fix adapted from https://github.com/thoughtbot/terrapin/pull/5 + +module Terrapin + module MultiPipeExtensions + def read + read_streams(@stdout_in, @stderr_in) + end + + def close_read + begin + @stdout_in.close + rescue IOError + # Do nothing + end + + begin + @stderr_in.close + rescue IOError + # Do nothing + end + end + + def read_streams(output, error) + @stdout_output = '' + @stderr_output = '' + + read_fds = [output, error] + + until read_fds.empty? + to_read, = IO.select(read_fds) + + if to_read.include?(output) + @stdout_output << read_stream(output) + read_fds.delete(output) if output.closed? + end + + if to_read.include?(error) + @stderr_output << read_stream(error) + read_fds.delete(error) if error.closed? + end + end + end + + def read_stream(io) + result = '' + + begin + while (partial_result = io.read_nonblock(8192)) + result << partial_result + end + rescue EOFError, Errno::EPIPE + io.close + rescue Errno::EINTR, Errno::EWOULDBLOCK, Errno::EAGAIN + # Do nothing + end + + result + end + end +end + +Terrapin::CommandLine::MultiPipe.prepend(Terrapin::MultiPipeExtensions) -- cgit From 6d9ad30bf861b2422c5951f7593a657675fedc24 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 23:46:59 +0200 Subject: Fix media redownload worker retrying on unexpected response codes (#16111) --- app/workers/redownload_media_worker.rb | 11 ++++++++++- lib/exceptions.rb | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb index 0638cd0f0..343caa32c 100644 --- a/app/workers/redownload_media_worker.rb +++ b/app/workers/redownload_media_worker.rb @@ -3,6 +3,7 @@ class RedownloadMediaWorker include Sidekiq::Worker include ExponentialBackoff + include JsonLdHelper sidekiq_options queue: 'pull', retry: 3 @@ -15,6 +16,14 @@ class RedownloadMediaWorker media_attachment.download_thumbnail! media_attachment.save rescue ActiveRecord::RecordNotFound - true + # Do nothing + rescue Mastodon::UnexpectedResponseError => e + response = e.response + + if response_error_unsalvageable?(response) + # Give up + else + raise e + end end end diff --git a/lib/exceptions.rb b/lib/exceptions.rb index 7c8e77871..eb472abaa 100644 --- a/lib/exceptions.rb +++ b/lib/exceptions.rb @@ -12,7 +12,11 @@ module Mastodon class RateLimitExceededError < Error; end class UnexpectedResponseError < Error + attr_reader :response + def initialize(response = nil) + @response = response + if response.respond_to? :uri super("#{response.uri} returned code #{response.code}") else -- cgit From 566fc909134586d1746ad60ee455832dec6bc61a Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 6 May 2021 14:22:54 +0200 Subject: Add Ruby 3.0 support (#16046) * Fix issues with POSIX::Spawn, Terrapin and Ruby 3.0 Also improve the Terrapin monkey-patch for the stderr/stdout issue. * Fix keyword argument handling throughout the codebase * Monkey-patch Paperclip to fix keyword arguments handling in validators * Change validation_extensions to please CodeClimate * Bump microformats from 4.2.1 to 4.3.1 * Allow Ruby 3.0 * Add Ruby 3.0 test target to CircleCI * Add test for admin dashboard warnings * Fix admin dashboard warnings on Ruby 3.0 --- .circleci/config.yml | 27 +++++++ Gemfile | 2 +- Gemfile.lock | 6 +- app/controllers/activitypub/outboxes_controller.rb | 2 +- app/controllers/api/v1/accounts_controller.rb | 4 +- .../api/v1/follow_requests_controller.rb | 2 +- app/models/session_activation.rb | 2 +- app/models/user.rb | 19 +++-- app/views/admin/dashboard/index.html.haml | 2 +- app/workers/import/relationship_worker.rb | 6 +- config/application.rb | 1 + config/initializers/session_store.rb | 5 +- lib/paperclip/validation_extensions.rb | 58 +++++++++++++++ lib/terrapin/multi_pipe_extensions.rb | 87 +++++++++++----------- .../controllers/admin/dashboard_controller_spec.rb | 12 ++- spec/mailers/notification_mailer_spec.rb | 4 +- spec/mailers/user_mailer_spec.rb | 4 +- spec/models/session_activation_spec.rb | 6 +- .../account_relationships_presenter_spec.rb | 2 +- 19 files changed, 177 insertions(+), 74 deletions(-) create mode 100644 lib/paperclip/validation_extensions.rb (limited to 'lib') diff --git a/.circleci/config.yml b/.circleci/config.yml index 862fa126b..2f3860d7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,13 @@ jobs: environment: *ruby_environment <<: *install_ruby_dependencies + install-ruby3.0: + <<: *defaults + docker: + - image: circleci/ruby:3.0-buster-node + environment: *ruby_environment + <<: *install_ruby_dependencies + build: <<: *defaults steps: @@ -187,6 +194,18 @@ jobs: - image: circleci/redis:5-alpine <<: *test_steps + test-ruby3.0: + <<: *defaults + docker: + - image: circleci/ruby:3.0-buster-node + environment: *ruby_environment + - image: circleci/postgres:12.2 + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + - image: circleci/redis:5-alpine + <<: *test_steps + test-webui: <<: *defaults docker: @@ -227,6 +246,10 @@ workflows: requires: - install - install-ruby2.7 + - install-ruby3.0: + requires: + - install + - install-ruby2.7 - build: requires: - install-ruby2.7 @@ -241,6 +264,10 @@ workflows: requires: - install-ruby2.6 - build + - test-ruby3.0: + requires: + - install-ruby3.0 + - build - test-webui: requires: - install diff --git a/Gemfile b/Gemfile index 6ca0a81de..5a55d6b04 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ # frozen_string_literal: true source 'https://rubygems.org' -ruby '>= 2.5.0', '< 3.0.0' +ruby '>= 2.5.0', '< 3.1.0' gem 'pkg-config', '~> 1.4' diff --git a/Gemfile.lock b/Gemfile.lock index b1ae4fd22..980750b63 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -292,7 +292,7 @@ GEM ipaddress (0.8.3) iso-639 (0.3.5) jmespath (1.4.0) - json (2.3.1) + json (2.5.1) json-canonicalization (0.2.1) json-ld (3.1.9) htmlentities (~> 4.3) @@ -344,7 +344,7 @@ GEM redis (>= 3.0.5) memory_profiler (1.0.0) method_source (1.0.0) - microformats (4.2.1) + microformats (4.3.1) json (~> 2.2) nokogiri (~> 1.10) mime-types (3.3.1) @@ -354,7 +354,7 @@ GEM nokogiri (~> 1) rake mini_mime (1.0.3) - mini_portile2 (2.5.0) + mini_portile2 (2.5.1) minitest (5.14.4) msgpack (1.4.2) multi_json (1.15.0) diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb index 5fd735ad6..111285036 100644 --- a/app/controllers/activitypub/outboxes_controller.rb +++ b/app/controllers/activitypub/outboxes_controller.rb @@ -20,7 +20,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController def outbox_presenter if page_requested? ActivityPub::CollectionPresenter.new( - id: outbox_url(page_params), + id: outbox_url(**page_params), type: :ordered, part_of: outbox_url, prev: prev_page, diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb index 996f1b79b..95869f554 100644 --- a/app/controllers/api/v1/accounts_controller.rb +++ b/app/controllers/api/v1/accounts_controller.rb @@ -35,7 +35,7 @@ class Api::V1::AccountsController < Api::BaseController follow = FollowService.new.call(current_user.account, @account, reblogs: params.key?(:reblogs) ? truthy_param?(:reblogs) : nil, notify: params.key?(:notify) ? truthy_param?(:notify) : nil, with_rate_limit: true) options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: follow.show_reblogs?, notify: follow.notify? } }, requested_map: { @account.id => false } } - render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(options) + render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(**options) end def block @@ -70,7 +70,7 @@ class Api::V1::AccountsController < Api::BaseController end def relationships(**options) - AccountRelationshipsPresenter.new([@account.id], current_user.account_id, options) + AccountRelationshipsPresenter.new([@account.id], current_user.account_id, **options) end def account_params diff --git a/app/controllers/api/v1/follow_requests_controller.rb b/app/controllers/api/v1/follow_requests_controller.rb index b34c76f29..f4b2a74d0 100644 --- a/app/controllers/api/v1/follow_requests_controller.rb +++ b/app/controllers/api/v1/follow_requests_controller.rb @@ -29,7 +29,7 @@ class Api::V1::FollowRequestsController < Api::BaseController end def relationships(**options) - AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, options) + AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, **options) end def load_accounts diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb index b0ce9d112..3a59bad93 100644 --- a/app/models/session_activation.rb +++ b/app/models/session_activation.rb @@ -44,7 +44,7 @@ class SessionActivation < ApplicationRecord end def activate(**options) - activation = create!(options) + activation = create!(**options) purge_old activation end diff --git a/app/models/user.rb b/app/models/user.rb index 0440627c5..4973c68b6 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -370,15 +370,20 @@ class User < ApplicationRecord protected - def send_devise_notification(notification, *args) + def send_devise_notification(notification, *args, **kwargs) # This method can be called in `after_update` and `after_commit` hooks, # but we must make sure the mailer is actually called *after* commit, # otherwise it may work on stale data. To do this, figure out if we are # within a transaction. + + # It seems like devise sends keyword arguments as a hash in the last + # positional argument + kwargs = args.pop if args.last.is_a?(Hash) && kwargs.empty? + if ActiveRecord::Base.connection.current_transaction.try(:records)&.include?(self) - pending_devise_notifications << [notification, args] + pending_devise_notifications << [notification, args, kwargs] else - render_and_send_devise_message(notification, *args) + render_and_send_devise_message(notification, *args, **kwargs) end end @@ -389,8 +394,8 @@ class User < ApplicationRecord end def send_pending_devise_notifications - pending_devise_notifications.each do |notification, args| - render_and_send_devise_message(notification, *args) + pending_devise_notifications.each do |notification, args, kwargs| + render_and_send_devise_message(notification, *args, **kwargs) end # Empty the pending notifications array because the @@ -403,8 +408,8 @@ class User < ApplicationRecord @pending_devise_notifications ||= [] end - def render_and_send_devise_message(notification, *args) - devise_mailer.send(notification, self, *args).deliver_later + def render_and_send_devise_message(notification, *args, **kwargs) + devise_mailer.send(notification, self, *args, **kwargs).deliver_later end def set_approved diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 57a753e6b..e8a2b46fd 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -5,7 +5,7 @@ .flash-message-stack - @system_checks.each do |message| .flash-message.warning - = t("admin.system_checks.#{message.key}.message_html", message.value ? { value: content_tag(:strong, message.value) } : {}) + = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil) - if message.action = link_to t("admin.system_checks.#{message.key}.action"), message.action diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb index 4a7100435..6791b15c3 100644 --- a/app/workers/import/relationship_worker.rb +++ b/app/workers/import/relationship_worker.rb @@ -5,7 +5,7 @@ class Import::RelationshipWorker sidekiq_options queue: 'pull', retry: 8, dead: false - def perform(account_id, target_account_uri, relationship, options = {}) + def perform(account_id, target_account_uri, relationship, options) from_account = Account.find(account_id) target_domain = domain(target_account_uri) target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) } @@ -16,7 +16,7 @@ class Import::RelationshipWorker case relationship when 'follow' begin - FollowService.new.call(from_account, target_account, options) + FollowService.new.call(from_account, target_account, **options) rescue ActiveRecord::RecordInvalid raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count end @@ -27,7 +27,7 @@ class Import::RelationshipWorker when 'unblock' UnblockService.new.call(from_account, target_account) when 'mute' - MuteService.new.call(from_account, target_account, options) + MuteService.new.call(from_account, target_account, **options) when 'unmute' UnmuteService.new.call(from_account, target_account) end diff --git a/config/application.rb b/config/application.rb index 37a996224..08a4e4c97 100644 --- a/config/application.rb +++ b/config/application.rb @@ -10,6 +10,7 @@ require_relative '../lib/exceptions' require_relative '../lib/enumerable' require_relative '../lib/sanitize_ext/sanitize_config' require_relative '../lib/redis/namespace_extensions' +require_relative '../lib/paperclip/validation_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/media_type_spoof_detector_extensions' diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index e5d1be4c6..3d9bf96fd 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -1,7 +1,6 @@ # Be sure to restart your server when you modify this file. -Rails.application.config.session_store :cookie_store, { +Rails.application.config.session_store :cookie_store, key: '_mastodon_session', secure: (Rails.env.production? || ENV['LOCAL_HTTPS'] == 'true'), - same_site: :lax, -} + same_site: :lax diff --git a/lib/paperclip/validation_extensions.rb b/lib/paperclip/validation_extensions.rb new file mode 100644 index 000000000..0df0434f6 --- /dev/null +++ b/lib/paperclip/validation_extensions.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility + +module Paperclip + module Validators + module AttachmentSizeValidatorExtensions + def validate_each(record, attr_name, _value) + base_attr_name = attr_name + attr_name = "#{attr_name}_file_size".to_sym + value = record.send(:read_attribute_for_validation, attr_name) + + if value.present? + options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value| + option_value = option_value.call(record) if option_value.is_a?(Proc) + option_value = extract_option_value(option, option_value) + + next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value) + + error_message_key = options[:in] ? :in_between : option + [attr_name, base_attr_name].each do |error_attr_name| + record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge( + min: min_value_in_human_size(record), + max: max_value_in_human_size(record), + count: human_size(option_value) + )) + end + end + end + end + end + + module AttachmentContentTypeValidatorExtensions + def mark_invalid(record, attribute, types) + record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') }) + end + end + + module AttachmentPresenceValidatorExtensions + def validate_each(record, attribute, _value) + if record.send("#{attribute}_file_name").blank? + record.errors.add(attribute, :blank, **options) + end + end + end + + module AttachmentFileNameValidatorExtensions + def mark_invalid(record, attribute, patterns) + record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') }) + end + end + end +end + +Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions) +Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions) +Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions) +Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions) diff --git a/lib/terrapin/multi_pipe_extensions.rb b/lib/terrapin/multi_pipe_extensions.rb index 51d7de37c..209f4ad6c 100644 --- a/lib/terrapin/multi_pipe_extensions.rb +++ b/lib/terrapin/multi_pipe_extensions.rb @@ -1,61 +1,64 @@ # frozen_string_literal: false -# Fix adapted from https://github.com/thoughtbot/terrapin/pull/5 + +require 'fcntl' module Terrapin module MultiPipeExtensions - def read - read_streams(@stdout_in, @stderr_in) - end + def initialize + @stdout_in, @stdout_out = IO.pipe + @stderr_in, @stderr_out = IO.pipe - def close_read - begin - @stdout_in.close - rescue IOError - # Do nothing - end - - begin - @stderr_in.close - rescue IOError - # Do nothing - end + clear_nonblocking_flags! end - def read_streams(output, error) - @stdout_output = '' - @stderr_output = '' + def pipe_options + # Add some flags to explicitly close the other end of the pipes + { out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close } + end - read_fds = [output, error] + def read + # While we are patching Terrapin, fix child process potentially getting stuck on writing + # to stderr. - until read_fds.empty? - to_read, = IO.select(read_fds) + @stdout_output = +'' + @stderr_output = +'' - if to_read.include?(output) - @stdout_output << read_stream(output) - read_fds.delete(output) if output.closed? - end + fds_to_read = [@stdout_in, @stderr_in] + until fds_to_read.empty? + rs, = IO.select(fds_to_read) - if to_read.include?(error) - @stderr_output << read_stream(error) - read_fds.delete(error) if error.closed? - end + read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in) + read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in) end end - def read_stream(io) - result = '' - - begin - while (partial_result = io.read_nonblock(8192)) - result << partial_result - end - rescue EOFError, Errno::EPIPE - io.close - rescue Errno::EINTR, Errno::EWOULDBLOCK, Errno::EAGAIN - # Do nothing + private + + # @param [IO] io IO Stream to read until there is nothing to read + # @param [String] result Mutable string to which read values will be appended to + # @param [Array] fds_to_read Mutable array from which `io` should be removed on EOF + def read_nonblocking!(io, result, fds_to_read) + while (partial_result = io.read_nonblock(8192)) + result << partial_result end + rescue IO::WaitReadable + # Do nothing + rescue EOFError + fds_to_read.delete(io) + end + + def clear_nonblocking_flags! + # Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as + # needed when calling fork/exec-related syscalls, but posix-spawn does + # not currently do that, so we need to do it manually for the time being + # so that the child process do not error out when the buffers are full. + stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL) + @stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK - result + stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL) + @stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK + rescue NameError, NotImplementedError, Errno::EINVAL + # Probably on windows, where pipes are blocking by default end end end diff --git a/spec/controllers/admin/dashboard_controller_spec.rb b/spec/controllers/admin/dashboard_controller_spec.rb index 73b50e721..7824854f9 100644 --- a/spec/controllers/admin/dashboard_controller_spec.rb +++ b/spec/controllers/admin/dashboard_controller_spec.rb @@ -3,9 +3,19 @@ require 'rails_helper' describe Admin::DashboardController, type: :controller do + render_views + describe 'GET #index' do - it 'returns 200' do + before do + allow(Admin::SystemCheck).to receive(:perform).and_return([ + Admin::SystemCheck::Message.new(:database_schema_check), + Admin::SystemCheck::Message.new(:rules_check, nil, admin_rules_path), + Admin::SystemCheck::Message.new(:sidekiq_process_check, 'foo, bar'), + ]) sign_in Fabricate(:user, admin: true) + end + + it 'returns 200' do get :index expect(response).to have_http_status(200) diff --git a/spec/mailers/notification_mailer_spec.rb b/spec/mailers/notification_mailer_spec.rb index 3ae106218..9b645bad8 100644 --- a/spec/mailers/notification_mailer_spec.rb +++ b/spec/mailers/notification_mailer_spec.rb @@ -10,12 +10,12 @@ RSpec.describe NotificationMailer, type: :mailer do it 'renders subject localized for the locale of the receiver' do locale = %i(de en).sample receiver.update!(locale: locale) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale)) end it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do receiver.update!(locale: nil) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale)) end end diff --git a/spec/mailers/user_mailer_spec.rb b/spec/mailers/user_mailer_spec.rb index 6b430b505..9c866788f 100644 --- a/spec/mailers/user_mailer_spec.rb +++ b/spec/mailers/user_mailer_spec.rb @@ -9,12 +9,12 @@ describe UserMailer, type: :mailer do it 'renders subject localized for the locale of the receiver' do locale = I18n.available_locales.sample receiver.update!(locale: locale) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale)) end it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do receiver.update!(locale: nil) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale)) end end diff --git a/spec/models/session_activation_spec.rb b/spec/models/session_activation_spec.rb index 2aa695037..450dc1399 100644 --- a/spec/models/session_activation_spec.rb +++ b/spec/models/session_activation_spec.rb @@ -74,13 +74,13 @@ RSpec.describe SessionActivation, type: :model do let(:options) { { user: Fabricate(:user), session_id: '1' } } it 'calls create! and purge_old' do - expect(described_class).to receive(:create!).with(options) + expect(described_class).to receive(:create!).with(**options) expect(described_class).to receive(:purge_old) - described_class.activate(options) + described_class.activate(**options) end it 'returns an instance of SessionActivation' do - expect(described_class.activate(options)).to be_kind_of SessionActivation + expect(described_class.activate(**options)).to be_kind_of SessionActivation end end diff --git a/spec/presenters/account_relationships_presenter_spec.rb b/spec/presenters/account_relationships_presenter_spec.rb index f8b048d38..edfbbb354 100644 --- a/spec/presenters/account_relationships_presenter_spec.rb +++ b/spec/presenters/account_relationships_presenter_spec.rb @@ -13,7 +13,7 @@ RSpec.describe AccountRelationshipsPresenter do allow(Account).to receive(:domain_blocking_map).with(account_ids, current_account_id).and_return(default_map) end - let(:presenter) { AccountRelationshipsPresenter.new(account_ids, current_account_id, options) } + let(:presenter) { AccountRelationshipsPresenter.new(account_ids, current_account_id, **options) } let(:current_account_id) { Fabricate(:account).id } let(:account_ids) { [Fabricate(:account).id] } let(:default_map) { { 1 => true } } -- cgit From a5f91a11d07db93526597131100c919f21a5dd78 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 7 May 2021 15:56:45 +0200 Subject: Fix older migrations on Ruby 3 (#16174) --- config/application.rb | 1 + lib/mastodon/migration_helpers.rb | 16 ++++++++-------- lib/paperclip/schema_extensions.rb | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 lib/paperclip/schema_extensions.rb (limited to 'lib') diff --git a/config/application.rb b/config/application.rb index 08a4e4c97..8200a2fcf 100644 --- a/config/application.rb +++ b/config/application.rb @@ -10,6 +10,7 @@ require_relative '../lib/exceptions' require_relative '../lib/enumerable' require_relative '../lib/sanitize_ext/sanitize_config' require_relative '../lib/redis/namespace_extensions' +require_relative '../lib/paperclip/schema_extensions' require_relative '../lib/paperclip/validation_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 147642a1c..521d903bf 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -95,7 +95,7 @@ module Mastodon allow_null: options[:null] ) else - add_column(table_name, column_name, :datetime_with_timezone, options) + add_column(table_name, column_name, :datetime_with_timezone, **options) end end end @@ -120,7 +120,7 @@ module Mastodon options = options.merge({ algorithm: :concurrently }) disable_statement_timeout - add_index(table_name, column_name, options) + add_index(table_name, column_name, **options) end # Removes an existed index, concurrently when supported @@ -144,7 +144,7 @@ module Mastodon disable_statement_timeout end - remove_index(table_name, options.merge({ column: column_name })) + remove_index(table_name, **options.merge({ column: column_name })) end # Removes an existing index, concurrently when supported @@ -168,7 +168,7 @@ module Mastodon disable_statement_timeout end - remove_index(table_name, options.merge({ name: index_name })) + remove_index(table_name, **options.merge({ name: index_name })) end # Only available on Postgresql >= 9.2 @@ -472,7 +472,7 @@ module Mastodon col_opts[:limit] = old_col.limit end - add_column(table, new, new_type, col_opts) + add_column(table, new, new_type, **col_opts) # We set the default value _after_ adding the column so we don't end up # updating any existing data with the default value. This isn't @@ -510,10 +510,10 @@ module Mastodon new_pk_index_name = "index_#{table}_on_#{column}_cm" unless indexes_for(table, column).find{|i| i.name == old_pk_index_name} - add_concurrent_index(table, [temp_column], { + add_concurrent_index(table, [temp_column], unique: true, name: new_pk_index_name - }) + ) end end end @@ -763,7 +763,7 @@ module Mastodon options[:using] = index.using if index.using options[:where] = index.where if index.where - add_concurrent_index(table, new_columns, options) + add_concurrent_index(table, new_columns, **options) end end diff --git a/lib/paperclip/schema_extensions.rb b/lib/paperclip/schema_extensions.rb new file mode 100644 index 000000000..8d065676a --- /dev/null +++ b/lib/paperclip/schema_extensions.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Monkey-patch various Paperclip methods for Ruby 3.0 compatibility + +module Paperclip + module Schema + module StatementsExtensions + def add_attachment(table_name, *attachment_names) + raise ArgumentError, 'Please specify attachment name in your add_attachment call in your migration.' if attachment_names.empty? + + options = attachment_names.extract_options! + + attachment_names.each do |attachment_name| + COLUMNS.each_pair do |column_name, column_type| + column_options = options.merge(options[column_name.to_sym] || {}) + add_column(table_name, "#{attachment_name}_#{column_name}", column_type, **column_options) + end + end + end + end + + module TableDefinitionExtensions + def attachment(*attachment_names) + options = attachment_names.extract_options! + attachment_names.each do |attachment_name| + COLUMNS.each_pair do |column_name, column_type| + column_options = options.merge(options[column_name.to_sym] || {}) + column("#{attachment_name}_#{column_name}", column_type, **column_options) + end + end + end + end + end +end + +Paperclip::Schema::Statements.prepend(Paperclip::Schema::StatementsExtensions) +Paperclip::Schema::TableDefinition.prepend(Paperclip::Schema::TableDefinitionExtensions) -- cgit From d1442a06c3c9e0a5ab2238ef2bef2cbfae79af7f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 8 May 2021 17:15:06 +0200 Subject: Bump version to 3.4.0rc1 (#16053) --- CHANGELOG.md | 143 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/mastodon/version.rb | 4 +- 2 files changed, 145 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d749c255..c393b8d6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,149 @@ Changelog All notable changes to this project will be documented in this file. +## Unreleased +### Added + +- **Add follow recommendations for onboarding** ([Gargron](https://github.com/tootsuite/mastodon/pull/15945), [Gargron](https://github.com/tootsuite/mastodon/pull/16161), [Gargron](https://github.com/tootsuite/mastodon/pull/16060), [Gargron](https://github.com/tootsuite/mastodon/pull/16077), [Gargron](https://github.com/tootsuite/mastodon/pull/16078), [Gargron](https://github.com/tootsuite/mastodon/pull/16160), [Gargron](https://github.com/tootsuite/mastodon/pull/16079), [noellabo](https://github.com/tootsuite/mastodon/pull/16044), [noellabo](https://github.com/tootsuite/mastodon/pull/16045), [Gargron](https://github.com/tootsuite/mastodon/pull/16152), [Gargron](https://github.com/tootsuite/mastodon/pull/16153), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16082), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16173), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16159)) + - Tutorial on first web UI launch has been replaced with follow suggestions + - Follow suggestions take user locale into account and are a mix of accounts most followed by currently active local users, and accounts that wrote the most shared/favourited posts in the last 30 days + - Only accounts that have opted-in to being discoverable from their profile settings, and that do not require follow requests, will be suggested + - Moderators can review suggestions for every supported locale and suppress specific suggestions from appearing and admins can ensure certain accounts always show up in suggestions from the settings area + - New users no longer automatically follow admins +- **Add server rules** ([Gargron](https://github.com/tootsuite/mastodon/pull/15769), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15778)) + - Admins can create and edit itemized server rules + - They are available through the REST API and on the about page +- **Add canonical e-mail blocks for suspended accounts** ([Gargron](https://github.com/tootsuite/mastodon/pull/16049)) + - Normally, people can make multiple accounts using the same e-mail address using the `+` trick or by inserting or removing `.` characters from the first part of their address + - Once an account is suspended, it will no longer be possible for the e-mail address used by that account to be used for new sign-ups in any of its forms +- Add management of delivery availability in admin UI ([noellabo](https://github.com/tootsuite/mastodon/pull/15771)) +- **Add system checks to dashboard in admin UI** ([Gargron](https://github.com/tootsuite/mastodon/pull/15989), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15954), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16002)) + - The dashboard will now warn you if you some Sidekiq queues are not being processed, if you have not defined any server rules, or if you forgot to run database migrations from the latest Mastodon upgrade +- Add inline description of moderation actions in admin UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15792)) +- Add "recommended" label to activity/peers API toggles in admin UI ([Gargron](https://github.com/tootsuite/mastodon/pull/16081)) +- Add joined date to profiles in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/16169)) +- Add transition to media modal background in web UI ([mkljczk](https://github.com/tootsuite/mastodon/pull/15843)) +- Add option to opt-out of unread notification markers in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15842)) +- Add borders to ๐Ÿ“ฑ, ๐Ÿšฒ, and ๐Ÿ“ฒ emojis in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15794), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16035)) +- Add dropdown for boost privacy in boost confirmation modal in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15704)) +- Add support for Ruby 3.0 ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16046), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16174)) +- Add `Message-ID` header to outgoing emails ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16076)) + - Some e-mail spam filters penalize e-mails that have a `Message-ID` header that uses a different domain name than the sending e-mail address. Now, the same domain will be used +- Add `af`, `gd` and `si` locales ([Gargron](https://github.com/tootsuite/mastodon/pull/16090)) +- Add guard against DNS rebinding attacks ([noellabo](https://github.com/tootsuite/mastodon/pull/16087), [noellabo](https://github.com/tootsuite/mastodon/pull/16095)) +- Add HTTP header to explicitly opt-out of FLoC by default ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16036)) +- Add missing push notification title for polls and statuses ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15929), [mkljczk](https://github.com/tootsuite/mastodon/pull/15564), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15931)) +- Add `POST /api/v1/emails/confirmations` to REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/15816), [Gargron](https://github.com/tootsuite/mastodon/pull/15949)) + - This method allows an app through which a user signed-up to request a new confirmation e-mail to be sent, or to change the e-mail of the account before it is confirmed +- Add `GET /api/v1/accounts/lookup` to REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/15740), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15750)) + - This method allows to quickly convert a username of a known account to an ID that can be used with the REST API, or to check if a username is available + for sign-up +- Add `policy` param to `POST /api/v1/push/subscriptions` in REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/16040)) + - This param allows an app to control from whom notifications should be delivered as push notifications to the app +- Add `details` to error response for `POST /api/v1/accounts` in REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/15803)) + - This attribute allows an app to display more helpful information to the user about why the sign-up did not succeed + +### Changed + +- Change trending hashtags to be affected be reblogs ([Gargron](https://github.com/tootsuite/mastodon/pull/16164)) + - Previously, only original posts contributed to a hashtag's trending score + - Now, reblogs of posts will also contribute to that hashtag's trending score +- Change e-mail confirmation link to always redirect to web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16151)) +- Change log level of worker lifecycle to WARN in streaming API ([Gargron](https://github.com/tootsuite/mastodon/pull/16110)) + - Since running with INFO log level in production is not always desirable, it is easy to miss when a worker is shutdown and a new one is started +- Change the nouns "toot" and "status" to "post" in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/16080), [Gargron](https://github.com/tootsuite/mastodon/pull/16089)) + - To be clear, the button still says "Toot!" +- Change order of dropdown menu on posts to be more intuitive in web UI ([ariasuni](https://github.com/tootsuite/mastodon/pull/15647)) +- Change description of keyboard shortcuts in web UI ([ariasuni](https://github.com/tootsuite/mastodon/pull/16129)) +- Change option labels on edit profile page ([Gargron](https://github.com/tootsuite/mastodon/pull/16041)) + - "Lock account" is now "Require follow requests" + - "List this account on the directory" is now "Suggest account to others" + - "Hide your network" is now "Hide your social graph" +- Change newly generated account IDs to not be enumerable ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15844)) +- Change Web Push API deliveries to use request pooling ([Gargron](https://github.com/tootsuite/mastodon/pull/16014)) +- Change multiple mentions with same username to render with domain ([Gargron](https://github.com/tootsuite/mastodon/pull/15718), [noellabo](https://github.com/tootsuite/mastodon/pull/16038)) + - When a post contains mentions of two or more users who have the same username, but on different domains, render their names with domain to help disambiguate them + - Always render the domain of usernames used in profile metadata +- Change health check endpoint to reveal less information ([Gargron](https://github.com/tootsuite/mastodon/pull/15988)) +- Change account counters to use upsert (requires Postgres >= 9.5) ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15913)) +- Change `mastodon:setup` to not call `assets:precompile` in Docker ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/13942)) +- **Change max. image dimensions to 1920x1080px (1080p)** ([Gargron](https://github.com/tootsuite/mastodon/pull/15690)) + - Previously, this was 1280x1280px + - This is the amount of pixels that original images get downsized to +- Change custom emoji to be animated when hovering container in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15637)) +- Change streaming API from deprecated ClusterWS/cws to ws ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15932)) +- Change systemd configuration to add sandboxing features ([Izorkin](https://github.com/tootsuite/mastodon/pull/15937), [Izorkin](https://github.com/tootsuite/mastodon/pull/16103), [Izorkin](https://github.com/tootsuite/mastodon/pull/16127)) +- Change nginx configuration to make running Onion service easier ([cohosh](https://github.com/tootsuite/mastodon/pull/15498)) +- Change Helm configuration ([dunn](https://github.com/tootsuite/mastodon/pull/15722), [dunn](https://github.com/tootsuite/mastodon/pull/15728), [dunn](https://github.com/tootsuite/mastodon/pull/15748), [dunn](https://github.com/tootsuite/mastodon/pull/15749), [dunn](https://github.com/tootsuite/mastodon/pull/15767)) +- Change Docker configuration ([SuperSandro2000](https://github.com/tootsuite/mastodon/pull/10823), [mashirozx](https://github.com/tootsuite/mastodon/pull/15978)) + +### Removed + +- Remove PubSubHubbub-related columns from accounts table ([Gargron](https://github.com/tootsuite/mastodon/pull/16170), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15857)) +- Remove dependency on @babel/plugin-proposal-class-properties ([ykzts](https://github.com/tootsuite/mastodon/pull/16155)) +- Remove dependency on pluck_each gem ([Gargron](https://github.com/tootsuite/mastodon/pull/16012)) +- Remove spam check and dependency on nilsimsa gem ([Gargron](https://github.com/tootsuite/mastodon/pull/16011)) +- Remove MySQL-specific code from Mastodon::MigrationHelpers ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15924)) +- Remove IE11 from supported browsers target ([gol-cha](https://github.com/tootsuite/mastodon/pull/15779)) + +### Fixed + +- Fix "You might be interested in" flashing while searching in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/16162)) +- Fix display of posts without text content in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15665)) +- Fix Google Translate breaking web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15610), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15611)) +- Fix web UI crashing when SVG support is disabled ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15809)) +- Fix web UI crash when a status opened in the media modal is deleted ([kaias1jp](https://github.com/tootsuite/mastodon/pull/15701)) +- Fix OCR language data failing to load in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15519)) +- Fix footer links not being clickable in Safari in web UI ([noellabo](https://github.com/tootsuite/mastodon/pull/15496)) +- Fix autofocus/autoselection not working on mobile in web UI ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15555), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15985)) +- Fix media redownload worker retrying on unexpected response codes ([Gargron](https://github.com/tootsuite/mastodon/pull/16111)) +- Fix thread resolve worker retrying when status no longer exists ([Gargron](https://github.com/tootsuite/mastodon/pull/16109)) +- Fix n+1 queries when rendering statuses in REST API ([abcang](https://github.com/tootsuite/mastodon/pull/15641)) +- Fix n+1 queries when rendering notifications in REST API ([abcang](https://github.com/tootsuite/mastodon/pull/15640)) +- Fix delete of local reply to local parent not being forwarded ([Gargron](https://github.com/tootsuite/mastodon/pull/16096)) +- Fix remote reporters not receiving suspend/unsuspend activities ([Gargron](https://github.com/tootsuite/mastodon/pull/16050)) +- Fix understanding (not fully qualified) `as:Public` and `Public` ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15948)) +- Fix actor update not being distributed on profile picture deletion ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15461)) +- Fix processing of incoming Delete activities ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16084)) +- Fix processing of incoming Block activities ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15546)) +- Fix processing of incoming Update activities of unknown accounts ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15514)) +- Fix URIs of repeat follow requests not being recorded ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15662)) +- Fix error on requests with no `Digest` header ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15782)) +- Fix activity object not requiring signature in secure mode ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15592)) +- Fix database serialization failure returning HTTP 500 ([Gargron](https://github.com/tootsuite/mastodon/pull/16101)) +- Fix media processing getting stuck on too much stdin/stderr ([Gargron](https://github.com/tootsuite/mastodon/pull/16136)) +- Fix some inefficient array manipulations ([007lva](https://github.com/tootsuite/mastodon/pull/15513), [007lva](https://github.com/tootsuite/mastodon/pull/15527)) +- Fix some inefficient regex matching ([007lva](https://github.com/tootsuite/mastodon/pull/15528)) +- Fix some inefficient SQL queries ([abcang](https://github.com/tootsuite/mastodon/pull/16104), [abcang](https://github.com/tootsuite/mastodon/pull/16106), [abcang](https://github.com/tootsuite/mastodon/pull/16105)) +- Fix trying to fetch key from empty URI when verifying HTTP signature ([Gargron](https://github.com/tootsuite/mastodon/pull/16100)) +- Fix `tootctl maintenance fix-duplicates` failures ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15923), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15515)) +- Fix error when removing status caused by race condition ([Gargron](https://github.com/tootsuite/mastodon/pull/16099)) +- Fix misspelled URLs character counting ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15382)) +- Fix Sidekiq hanging forever due to a Resolv bug in Ruby 2.7.3 ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16157)) +- Fix edge case where follow limit interferes with accepting a follow ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16098)) +- Fix inconsistent lead text style in admin UI ([Gargron](https://github.com/tootsuite/mastodon/pull/16052), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16086)) +- Fix reports of already suspended accounts being recorded ([Gargron](https://github.com/tootsuite/mastodon/pull/16047)) +- Fix sign-up restrictions based on IP addresses not being enforced ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15607)) +- Fix YouTube embeds failing due to YouTube serving wrong OEmbed URLs ([Gargron](https://github.com/tootsuite/mastodon/pull/15716)) +- Fix error when rendering public pages with media without meta ([Gargron](https://github.com/tootsuite/mastodon/pull/16112)) +- Fix misaligned logo on follow button on public pages ([noellabo](https://github.com/tootsuite/mastodon/pull/15458)) +- Fix video modal not working on public pages ([noellabo](https://github.com/tootsuite/mastodon/pull/15469)) +- Fix race conditions on account migration creation ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15597)) +- Fix not being able to change world filter expiration back to โ€œNeverโ€ ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15858)) +- Fix `.env.vagrant` not setting `RAILS_ENV` variable ([chandrn7](https://github.com/tootsuite/mastodon/pull/15709)) +- Fix error when muting users with `duration` in REST API ([Tak](https://github.com/tootsuite/mastodon/pull/15516)) +- Fix border padding on front page in light theme ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15926)) +- Fix wrong URL to custom CSS when `CDN_HOST` is used ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15927)) +- Fix `tootctl accounts unfollow` ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15639)) +- Fix `tootctl emoji import` wasting time on MacOS shadow files ([cortices](https://github.com/tootsuite/mastodon/pull/15430)) +- Fix `tootctl emoji import` not treating shortcodes as case-insensitive ([angristan](https://github.com/tootsuite/mastodon/pull/15738)) +- Fix some issues with SAML account creation ([Gargron](https://github.com/tootsuite/mastodon/pull/15222), [kaiyou](https://github.com/tootsuite/mastodon/pull/15511)) +- Fix MX validation applying for explicitly allowed e-mail domains ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15930)) +- Fix share page not using configured custom mascot ([tribela](https://github.com/tootsuite/mastodon/pull/15687)) +- Fix instance actor not being automatically created if it wasn't seeded properly ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15693)) +- Fix HTTPS enforcement preventing Mastodon from being run as an Onion service ([cohosh](https://github.com/tootsuite/mastodon/pull/15560), [jtracey](https://github.com/tootsuite/mastodon/pull/15741), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15712), [cohosh](https://github.com/tootsuite/mastodon/pull/15725)) +- Fix app name, website and redirect URIs not having a maximum length ([Gargron](https://github.com/tootsuite/mastodon/pull/16042)) + ## [3.3.0] - 2020-12-27 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index bd0915775..0a0a7878b 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,7 +9,7 @@ module Mastodon end def minor - 3 + 4 end def patch @@ -17,7 +17,7 @@ module Mastodon end def flags - '' + 'rc1' end def suffix -- cgit From 9b18914c3572be5b661e4d8e31578fae4ee347be Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Sun, 9 May 2021 17:40:17 +0900 Subject: Add a Redis environment variable for sidekiq (#16188) --- config/initializers/sidekiq.rb | 11 ++--------- lib/mastodon/redis_config.rb | 12 ++++++++++-- 2 files changed, 12 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index fc85a3913..219554df4 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,14 +1,7 @@ # frozen_string_literal: true -namespace = ENV.fetch('REDIS_NAMESPACE') { nil } -redis_params = { url: ENV['REDIS_URL'], driver: :hiredis } - -if namespace - redis_params[:namespace] = namespace -end - Sidekiq.configure_server do |config| - config.redis = redis_params + config.redis = REDIS_SIDEKIQ_PARAMS config.server_middleware do |chain| chain.add SidekiqErrorHandler @@ -26,7 +19,7 @@ Sidekiq.configure_server do |config| end Sidekiq.configure_client do |config| - config.redis = redis_params + config.redis = REDIS_SIDEKIQ_PARAMS config.client_middleware do |chain| chain.add SidekiqUniqueJobs::Middleware::Client diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index d341a8484..5bfd26e34 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -22,9 +22,11 @@ end setup_redis_env_url setup_redis_env_url(:cache, false) +setup_redis_env_url(:sidekiq, false) -namespace = ENV.fetch('REDIS_NAMESPACE', nil) -cache_namespace = namespace ? namespace + '_cache' : 'cache' +namespace = ENV.fetch('REDIS_NAMESPACE', nil) +cache_namespace = namespace ? namespace + '_cache' : 'cache' +sidekiq_namespace = namespace REDIS_CACHE_PARAMS = { driver: :hiredis, @@ -32,3 +34,9 @@ REDIS_CACHE_PARAMS = { expires_in: 10.minutes, namespace: cache_namespace, }.freeze + +REDIS_SIDEKIQ_PARAMS = { + driver: :hiredis, + url: ENV['SIDEKIQ_REDIS_URL'], + namespace: sidekiq_namespace, +}.freeze -- cgit