Merge pull request #674 from ThibG/glitch-soc/merge-upstream

Merge upstream changes

5 files changed, 164 insertions, 47 deletions

diff --git a/lib/cli.rb b/lib/cli.rb
new file mode 100644
index 000000000..c7dae0276
--- /dev/null
+++ b/lib/cli.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative 'mastodon/media_cli'
+require_relative 'mastodon/emoji_cli'
+
+module Mastodon
+  class CLI < Thor
+    desc 'media SUBCOMMAND ...ARGS', 'manage media files'
+    subcommand 'media', Mastodon::MediaCLI
+
+    desc 'emoji SUBCOMMAND ...ARGS', 'manage custom emoji'
+    subcommand 'emoji', Mastodon::EmojiCLI
+  end
+end
diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb
new file mode 100644
index 000000000..8c4d9731c
--- /dev/null
+++ b/lib/mastodon/cli_helper.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+dev_null = Logger.new('/dev/null')
+
+Rails.logger                 = dev_null
+ActiveRecord::Base.logger    = dev_null
+HttpLog.configuration.logger = dev_null
+Paperclip.options[:log]      = false
diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb
new file mode 100644
index 000000000..71f8b2cc7
--- /dev/null
+++ b/lib/mastodon/emoji_cli.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+# rubocop:disable Rails/Output
+
+module Mastodon
+  class EmojiCLI < Thor
+    option :prefix
+    option :suffix
+    option :overwrite, type: :boolean
+    option :unlisted, type: :boolean
+    desc 'import PATH', 'import emoji from a TAR archive at PATH'
+    long_desc <<-LONG_DESC
+      Imports custom emoji from a TAR archive specified by PATH.
+
+      Existing emoji will be skipped unless the --overwrite option
+      is provided, in which case they will be overwritten.
+
+      With the --prefix option, a prefix can be added to all
+      generated shortcodes. Likewise, the --suffix option controls
+      the suffix of all shortcodes.
+
+      With the --unlisted option, the processed emoji will not be
+      visible in the emoji picker (but still usable via other means)
+    LONG_DESC
+    def import(path)
+      imported = 0
+      skipped  = 0
+      failed   = 0
+
+      Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
+        tar.each do |entry|
+          next unless entry.file? && entry.full_name.end_with?('.png')
+
+          shortcode    = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join
+          custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode)
+
+          if custom_emoji && !options[:overwrite]
+            skipped += 1
+            next
+          end
+
+          custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil)
+          custom_emoji.image = StringIO.new(entry.read)
+          custom_emoji.image_file_name = File.basename(entry.full_name)
+          custom_emoji.visible_in_picker = !options[:unlisted]
+
+          if custom_emoji.save
+            imported += 1
+          else
+            failed += 1
+            say('Failure/Error: ', :red)
+            say(entry.full_name)
+            say('    ' + custom_emoji.errors[:image].join(', '), :red)
+          end
+        end
+      end
+
+      puts
+      say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
+    end
+
+    private
+
+    def color(green, _yellow, red)
+      if !green.zero? && red.zero?
+        :green
+      elsif red.zero?
+        :yellow
+      else
+        :red
+      end
+    end
+  end
+end
+
+# rubocop:enable Rails/Output
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
new file mode 100644
index 000000000..00bd662f4
--- /dev/null
+++ b/lib/mastodon/media_cli.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+# rubocop:disable Rails/Output
+
+module Mastodon
+  class MediaCLI < Thor
+    option :days, type: :numeric, default: 7
+    option :background, type: :boolean, default: false
+    desc 'remove', 'remove remote media files'
+    long_desc <<-DESC
+      Removes locally cached copies of media attachments from other servers.
+
+      The --days option specifies how old media attachments have to be before
+      they are removed. It defaults to 7 days.
+
+      With the --background option, instead of deleting the files sequentially,
+      they will be queued into Sidekiq and the command will exit as soon as
+      possible. In Sidekiq they will be processed with higher concurrency, but
+      it may impact other operations of the Mastodon server, and it may overload
+      the underlying file storage.
+    DESC
+    def remove
+      time_ago  = options[:days].days.ago
+      queued    = 0
+      processed = 0
+
+      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
+        if options[:background]
+          queued += media_attachments.size
+          Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
+        else
+          media_attachments.each do |m|
+            Maintenance::UncacheMediaWorker.new.perform(m)
+            say('.', :green, false)
+            processed += 1
+          end
+        end
+      end
+
+      say
+
+      if options[:background]
+        say("Scheduled the deletion of #{queued} media attachments", :green)
+      else
+        say("Removed #{processed} media attachments", :green)
+      end
+    end
+  end
+end
+
+# rubocop:enable Rails/Output
diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index 191ce634c..7455478b6 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -222,7 +222,7 @@ namespace :mastodon do
         end
 
         if prompt.yes?('Do you want to access the uploaded files from your own domain?')
-          env['S3_CLOUDFRONT_HOST'] = prompt.ask('Domain for uploaded files:') do |q|
+          env['S3_ALIAS_HOST'] = prompt.ask('Domain for uploaded files:') do |q|
             q.required true
             q.default "files.#{env['LOCAL_DOMAIN']}"
             q.modify :strip
@@ -394,12 +394,6 @@ namespace :mastodon do
     end
   end
 
-  desc 'Execute daily tasks (deprecated)'
-  task :daily do
-    # No-op
-    # All of these tasks are now executed via sidekiq-scheduler
-  end
-
   desc 'Turn a user into an admin, identified by the USERNAME environment variable'
   task make_admin: :environment do
     include RoutingHelper
@@ -494,12 +488,6 @@ namespace :mastodon do
   end
 
   namespace :media do
-    desc 'Removes media attachments that have not been assigned to any status for longer than a day (deprecated)'
-    task clear: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-
     desc 'Remove media attachments attributed to silenced accounts'
     task remove_silenced: :environment do
       nb_media_attachments = 0
@@ -512,14 +500,10 @@ namespace :mastodon do
 
     desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
     task remove_remote: :environment do
-      time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
-      nb_media_attachments = 0
-
-      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
-        nb_media_attachments += media_attachments.length
-        Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
-      end
-      puts "Scheduled the deletion of #{nb_media_attachments} media attachments"
+      puts 'Please use `./bin/tootctl media remove --help` directly'.colorize(:yellow)
+      require_relative '../mastodon/media_cli'
+      cli = Mastodon::MediaCLI.new([], days: (ENV['NUM_DAYS'] || 7).to_i)
+      cli.invoke(:remove)
     end
 
     desc 'Set unknown attachment type for remote-only attachments'
@@ -548,21 +532,9 @@ namespace :mastodon do
     task clear: :environment do
       Pubsubhubbub::UnsubscribeWorker.push_bulk(Account.remote.without_followers.where.not(subscription_expires_at: nil).pluck(:id))
     end
-
-    desc 'Re-subscribes to soon expiring PuSH subscriptions (deprecated)'
-    task refresh: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
   end
 
   namespace :feeds do
-    desc 'Clear timelines of inactive users (deprecated)'
-    task clear: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-
     desc 'Clear all timelines without regenerating them'
     task clear_all: :environment do
       Redis.current.keys('feed:*').each { |key| Redis.current.del(key) }
@@ -576,21 +548,7 @@ namespace :mastodon do
     end
   end
 
-  namespace :emails do
-    desc 'Send out digest e-mails (deprecated)'
-    task digest: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-  end
-
   namespace :users do
-    desc 'Clear out unconfirmed users (deprecated)'
-    task clear: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-
     desc 'List e-mails of all admin users'
     task admins: :environment do
       puts 'Admin user emails:'