Merge pull request #1228 from ThibG/glitch-soc/merge-upstream

Merge upstream changes

6 files changed, 160 insertions, 12 deletions

diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb
new file mode 100644
index 000000000..3e54326ba
--- /dev/null
+++ b/lib/chewy/strategy/custom_sidekiq.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class CustomSidekiq < Base
+      class Worker
+        include ::Sidekiq::Worker
+
+        sidekiq_options queue: 'pull'
+
+        def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          type.constantize.import!(ids, options)
+        end
+      end
+
+      def update(type, objects, _options = {})
+        return unless Chewy.enabled?
+
+        ids = type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
+
+        return if ids.empty?
+
+        Worker.perform_async(type.name, ids)
+      end
+
+      def leave; end
+    end
+  end
+end
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index 8e52de1c3..b5435bb5e 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -17,7 +17,7 @@ module Mastodon
     option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean
     option :whitelist_mode, type: :boolean
-    desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
+    desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace'
     long_desc <<-LONG_DESC
       Remove all accounts from a given DOMAIN without leaving behind any
       records. Unlike a suspension, if the DOMAIN still exists in the wild,
@@ -27,16 +27,16 @@ module Mastodon
       from a single domain, all accounts from domains that are not whitelisted
       are removed from the database.
     LONG_DESC
-    def purge(domain = nil)
+    def purge(*domains)
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
       scope = begin
         if options[:whitelist_mode]
           Account.remote.where.not(domain: DomainAllow.pluck(:domain))
-        elsif domain.present?
-          Account.remote.where(domain: domain)
+        elsif !domains.empty?
+          Account.remote.where(domain: domains)
         else
-          say('No domain given', :red)
+          say('No domain(s) given', :red)
           exit(1)
         end
       end
@@ -45,11 +45,11 @@ module Mastodon
         SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run]
       end
 
-      DomainBlock.where(domain: domain).destroy_all unless options[:dry_run]
+      DomainBlock.where(domain: domains).destroy_all unless options[:dry_run]
 
       say("Removed #{processed} accounts#{dry_run}", :green)
 
-      custom_emojis = CustomEmoji.where(domain: domain)
+      custom_emojis = CustomEmoji.where(domain: domains)
       custom_emojis_count = custom_emojis.count
       custom_emojis.destroy_all unless options[:dry_run]
 
diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb
index 42ad93f1e..8bd5f9543 100644
--- a/lib/mastodon/search_cli.rb
+++ b/lib/mastodon/search_cli.rb
@@ -6,6 +6,7 @@ require_relative 'cli_helper'
 
 module Mastodon
   class SearchCLI < Thor
+    option :processes, default: 2, aliases: [:p]
     desc 'deploy', 'Create or update an ElasticSearch index and populate it'
     long_desc <<~LONG_DESC
       If ElasticSearch is empty, this command will create the necessary indices
@@ -13,10 +14,28 @@ module Mastodon
 
       This command will also upgrade indices if the underlying schema has been
       changed since the last run.
+
+      With the --processes option, parallelize execution of the command. The
+      default is 2. If "auto" is specified, the number is automatically
+      derived from available CPUs.
     LONG_DESC
     def deploy
-      processed = Chewy::RakeHelper.upgrade
-      Chewy::RakeHelper.sync(except: processed)
+      processed = Chewy::RakeHelper.upgrade(parallel: processes)
+      Chewy::RakeHelper.sync(except: processed, parallel: processes)
+    end
+
+    private
+
+    def processes
+      return true if options[:processes] == 'auto'
+
+      num = options[:processes].to_i
+
+      if num < 2
+        nil
+      else
+        num
+      end
     end
   end
 end
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 38cae8766..8010e526b 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -17,7 +17,7 @@ module Mastodon
     end
 
     def flags
-      'rc2'
+      ''
     end
 
     def suffix
diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb
index cbab6fd99..64f12f963 100644
--- a/lib/paperclip/gif_transcoder.rb
+++ b/lib/paperclip/gif_transcoder.rb
@@ -1,5 +1,103 @@
 # frozen_string_literal: true
 
+class GifReader
+  attr_reader :animated
+
+  EXTENSION_LABELS = [0xf9, 0x01, 0xff].freeze
+  GIF_HEADERS      = %w(GIF87a GIF89a).freeze
+
+  class GifReaderException; end
+
+  class UnknownImageType < GifReaderException; end
+
+  class CannotParseImage < GifReaderException; end
+
+  def self.animated?(path)
+    new(path).animated
+  rescue GifReaderException
+    false
+  end
+
+  def initialize(path, max_frames = 2)
+    @path      = path
+    @nb_frames = 0
+
+    File.open(path, 'rb') do |s|
+      raise UnknownImageType unless GIF_HEADERS.include?(s.read(6))
+
+      # Skip to "packed byte"
+      s.seek(4, IO::SEEK_CUR)
+
+      # "Packed byte" gives us the size of the GIF color table
+      packed_byte, = s.read(1).unpack('C')
+
+      # Skip background color and aspect ratio
+      s.seek(2, IO::SEEK_CUR)
+
+      if packed_byte & 0x80 != 0
+        # GIF uses a global color table, skip it
+        s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR)
+      end
+
+      # Now read data
+      while @nb_frames < max_frames
+        separator = s.read(1)
+
+        case separator
+        when ',' # Image block
+          @nb_frames += 1
+
+          # Skip to "packed byte"
+          s.seek(8, IO::SEEK_CUR)
+          packed_byte, = s.read(1).unpack('C')
+
+          if packed_byte & 0x80 != 0
+            # Image uses a local color table, skip it
+            s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR)
+          end
+
+          # Skip lzw min code size
+          raise InvalidValue unless s.read(1).unpack('C')[0] >= 2
+
+          # Skip image data sub-blocks
+          skip_sub_blocks!(s)
+        when '!' # Extension block
+          skip_extension_block!(s)
+        when ';' # Trailer
+          break
+        else
+          raise CannotParseImage
+        end
+      end
+    end
+
+    @animated = @nb_frames > 1
+  end
+
+  private
+
+  def skip_extension_block!(file)
+    if EXTENSION_LABELS.include?(file.read(1).unpack('C')[0])
+      block_size, = file.read(1).unpack('C')
+      file.seek(block_size, IO::SEEK_CUR)
+    end
+
+    # Read until extension block end marker
+    skip_sub_blocks!(file)
+  end
+
+  # Skip sub-blocks up until block end marker
+  def skip_sub_blocks!(file)
+    loop do
+      size, = file.read(1).unpack('C')
+
+      break if size.zero?
+
+      file.seek(size, IO::SEEK_CUR)
+    end
+  end
+end
+
 module Paperclip
   # This transcoder is only to be used for the MediaAttachment model
   # to convert animated gifs to webm
@@ -19,8 +117,7 @@ module Paperclip
     private
 
     def needs_convert?
-      num_frames = identify('-format %n :file', file: file.path).to_i
-      options[:style] == :original && num_frames > 1
+      options[:style] == :original && GifReader.animated?(file.path)
     end
   end
 end
diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb
index c3504c17c..66f7feda5 100644
--- a/lib/paperclip/video_transcoder.rb
+++ b/lib/paperclip/video_transcoder.rb
@@ -6,7 +6,9 @@ module Paperclip
   class VideoTranscoder < Paperclip::Processor
     def make
       meta = ::Av.cli.identify(@file.path)
+
       attachment.instance.type = MediaAttachment.types[:gifv] unless meta[:audio_encode]
+      options[:format] = File.extname(attachment.instance.file_file_name)[1..-1] if options[:keep_same_format]
 
       Paperclip::Transcoder.make(file, options, attachment)
     end