Merge branch 'glitch'

8 files changed, 238 insertions, 67 deletions

diff --git a/lib/cli.rb b/lib/cli.rb
new file mode 100644
index 000000000..60bff4147
--- /dev/null
+++ b/lib/cli.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative 'mastodon/media_cli'
+require_relative 'mastodon/emoji_cli'
+require_relative 'mastodon/accounts_cli'
+module Mastodon
+  class CLI < Thor
+    desc 'media SUBCOMMAND ...ARGS', 'Manage media files'
+    subcommand 'media', Mastodon::MediaCLI
+
+    desc 'emoji SUBCOMMAND ...ARGS', 'Manage custom emoji'
+    subcommand 'emoji', Mastodon::EmojiCLI
+
+    desc 'accounts SUBCOMMAND ...ARGS', 'Manage accounts'
+    subcommand 'accounts', Mastodon::AccountsCLI
+  end
+end
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
new file mode 100644
index 000000000..83b69549d
--- /dev/null
+++ b/lib/mastodon/accounts_cli.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+module Mastodon
+  class AccountsCLI < Thor
+    option :all, type: :boolean
+    desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
+    long_desc <<-LONG_DESC
+      Generate and broadcast new RSA keys as part of security
+      maintenance.
+
+      With the --all option, all local accounts will be subject
+      to the rotation. Otherwise, and by default, only a single
+      account specified by the USERNAME argument will be
+      processed.
+    LONG_DESC
+    def rotate(username = nil)
+      if options[:all]
+        processed = 0
+        delay     = 0
+
+        Account.local.without_suspended.find_in_batches do |accounts|
+          accounts.each do |account|
+            rotate_keys_for_account(account, delay)
+            processed += 1
+            say('.', :green, false)
+          end
+
+          delay += 5.minutes
+        end
+
+        say
+        say("OK, rotated keys for #{processed} accounts", :green)
+      elsif username.present?
+        rotate_keys_for_account(Account.find_local(username))
+        say('OK', :green)
+      else
+        say('No account(s) given', :red)
+      end
+    end
+
+    private
+
+    def rotate_keys_for_account(account, delay = 0)
+      old_key = account.private_key
+      new_key = OpenSSL::PKey::RSA.new(2048).to_pem
+      account.update(private_key: new_key)
+      ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, sign_with: old_key)
+    end
+  end
+end
diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb
new file mode 100644
index 000000000..8c4d9731c
--- /dev/null
+++ b/lib/mastodon/cli_helper.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+dev_null = Logger.new('/dev/null')
+
+Rails.logger                 = dev_null
+ActiveRecord::Base.logger    = dev_null
+HttpLog.configuration.logger = dev_null
+Paperclip.options[:log]      = false
diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb
new file mode 100644
index 000000000..0a773c771
--- /dev/null
+++ b/lib/mastodon/emoji_cli.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+# rubocop:disable Rails/Output
+
+module Mastodon
+  class EmojiCLI < Thor
+    option :prefix
+    option :suffix
+    option :overwrite, type: :boolean
+    option :unlisted, type: :boolean
+    desc 'import PATH', 'Import emoji from a TAR archive at PATH'
+    long_desc <<-LONG_DESC
+      Imports custom emoji from a TAR archive specified by PATH.
+
+      Existing emoji will be skipped unless the --overwrite option
+      is provided, in which case they will be overwritten.
+
+      With the --prefix option, a prefix can be added to all
+      generated shortcodes. Likewise, the --suffix option controls
+      the suffix of all shortcodes.
+
+      With the --unlisted option, the processed emoji will not be
+      visible in the emoji picker (but still usable via other means)
+    LONG_DESC
+    def import(path)
+      imported = 0
+      skipped  = 0
+      failed   = 0
+
+      Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
+        tar.each do |entry|
+          next unless entry.file? && entry.full_name.end_with?('.png')
+
+          shortcode    = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join
+          custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode)
+
+          if custom_emoji && !options[:overwrite]
+            skipped += 1
+            next
+          end
+
+          custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil)
+          custom_emoji.image = StringIO.new(entry.read)
+          custom_emoji.image_file_name = File.basename(entry.full_name)
+          custom_emoji.visible_in_picker = !options[:unlisted]
+
+          if custom_emoji.save
+            imported += 1
+          else
+            failed += 1
+            say('Failure/Error: ', :red)
+            say(entry.full_name)
+            say('    ' + custom_emoji.errors[:image].join(', '), :red)
+          end
+        end
+      end
+
+      puts
+      say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
+    end
+
+    private
+
+    def color(green, _yellow, red)
+      if !green.zero? && red.zero?
+        :green
+      elsif red.zero?
+        :yellow
+      else
+        :red
+      end
+    end
+  end
+end
+
+# rubocop:enable Rails/Output
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
new file mode 100644
index 000000000..ee28270da
--- /dev/null
+++ b/lib/mastodon/media_cli.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+# rubocop:disable Rails/Output
+
+module Mastodon
+  class MediaCLI < Thor
+    option :days, type: :numeric, default: 7
+    option :background, type: :boolean, default: false
+    desc 'remove', 'Remove remote media files'
+    long_desc <<-DESC
+      Removes locally cached copies of media attachments from other servers.
+
+      The --days option specifies how old media attachments have to be before
+      they are removed. It defaults to 7 days.
+
+      With the --background option, instead of deleting the files sequentially,
+      they will be queued into Sidekiq and the command will exit as soon as
+      possible. In Sidekiq they will be processed with higher concurrency, but
+      it may impact other operations of the Mastodon server, and it may overload
+      the underlying file storage.
+    DESC
+    def remove
+      time_ago  = options[:days].days.ago
+      queued    = 0
+      processed = 0
+
+      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
+        if options[:background]
+          queued += media_attachments.size
+          Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
+        else
+          media_attachments.each do |m|
+            Maintenance::UncacheMediaWorker.new.perform(m)
+            say('.', :green, false)
+            processed += 1
+          end
+        end
+      end
+
+      say
+
+      if options[:background]
+        say("Scheduled the deletion of #{queued} media attachments", :green)
+      else
+        say("Removed #{processed} media attachments", :green)
+      end
+    end
+  end
+end
+
+# rubocop:enable Rails/Output
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 1aa9b413e..d07108ad4 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -9,11 +9,11 @@ module Mastodon
     end
 
     def minor
-      4
+      5
     end
 
     def patch
-      3
+      0
     end
 
     def pre
@@ -21,7 +21,7 @@ module Mastodon
     end
 
     def flags
-      ''
+      'rc1'
     end
 
     def to_a
diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index 32039c31d..b76e90131 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -18,7 +18,7 @@ def each_schema_load_environment
   #    needing to do the same, and we can't even use the same method
   #    to do it.
 
-  if Rails.env == 'development'
+  if Rails.env.development?
     test_conf = ActiveRecord::Base.configurations['test']
 
     if test_conf['database']&.present?
diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index 191ce634c..649a22a0b 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -222,7 +222,7 @@ namespace :mastodon do
         end
 
         if prompt.yes?('Do you want to access the uploaded files from your own domain?')
-          env['S3_CLOUDFRONT_HOST'] = prompt.ask('Domain for uploaded files:') do |q|
+          env['S3_ALIAS_HOST'] = prompt.ask('Domain for uploaded files:') do |q|
             q.required true
             q.default "files.#{env['LOCAL_DOMAIN']}"
             q.modify :strip
@@ -280,14 +280,14 @@ namespace :mastodon do
 
         begin
           ActionMailer::Base.smtp_settings = {
-            :port                 => env['SMTP_PORT'],
-            :address              => env['SMTP_SERVER'],
-            :user_name            => env['SMTP_LOGIN'].presence,
-            :password             => env['SMTP_PASSWORD'].presence,
-            :domain               => env['LOCAL_DOMAIN'],
-            :authentication       => env['SMTP_AUTH_METHOD'] == 'none' ? nil : env['SMTP_AUTH_METHOD'] || :plain,
-            :openssl_verify_mode  => env['SMTP_OPENSSL_VERIFY_MODE'],
-            :enable_starttls_auto => true,
+            port:                 env['SMTP_PORT'],
+            address:              env['SMTP_SERVER'],
+            user_name:            env['SMTP_LOGIN'].presence,
+            password:             env['SMTP_PASSWORD'].presence,
+            domain:               env['LOCAL_DOMAIN'],
+            authentication:       env['SMTP_AUTH_METHOD'] == 'none' ? nil : env['SMTP_AUTH_METHOD'] || :plain,
+            openssl_verify_mode:  env['SMTP_OPENSSL_VERIFY_MODE'],
+            enable_starttls_auto: true,
           }
 
           ActionMailer::Base.default_options = {
@@ -326,13 +326,11 @@ namespace :mastodon do
 
         if prompt.yes?('Prepare the database now?')
           prompt.say 'Running `RAILS_ENV=production rails db:setup` ...'
-          prompt.say "\n"
+          prompt.say "\n\n"
 
           if cmd.run!({ RAILS_ENV: 'production', SAFETY_ASSURED: 1 }, :rails, 'db:setup').failure?
-            prompt.say "\n"
             prompt.error 'That failed! Perhaps your configuration is not right'
           else
-            prompt.say "\n"
             prompt.ok 'Done!'
           end
         end
@@ -343,13 +341,11 @@ namespace :mastodon do
 
         if prompt.yes?('Compile the assets now?')
           prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...'
-          prompt.say "\n"
+          prompt.say "\n\n"
 
           if cmd.run!({ RAILS_ENV: 'production' }, :rails, 'assets:precompile').failure?
-            prompt.say "\n"
             prompt.error 'That failed! Maybe you need swap space?'
           else
-            prompt.say "\n"
             prompt.say 'Done!'
           end
         end
@@ -394,12 +390,6 @@ namespace :mastodon do
     end
   end
 
-  desc 'Execute daily tasks (deprecated)'
-  task :daily do
-    # No-op
-    # All of these tasks are now executed via sidekiq-scheduler
-  end
-
   desc 'Turn a user into an admin, identified by the USERNAME environment variable'
   task make_admin: :environment do
     include RoutingHelper
@@ -494,12 +484,6 @@ namespace :mastodon do
   end
 
   namespace :media do
-    desc 'Removes media attachments that have not been assigned to any status for longer than a day (deprecated)'
-    task clear: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-
     desc 'Remove media attachments attributed to silenced accounts'
     task remove_silenced: :environment do
       nb_media_attachments = 0
@@ -512,14 +496,10 @@ namespace :mastodon do
 
     desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
     task remove_remote: :environment do
-      time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
-      nb_media_attachments = 0
-
-      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
-        nb_media_attachments += media_attachments.length
-        Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
-      end
-      puts "Scheduled the deletion of #{nb_media_attachments} media attachments"
+      puts 'Please use `./bin/tootctl media remove --help` directly'.colorize(:yellow)
+      require_relative '../mastodon/media_cli'
+      cli = Mastodon::MediaCLI.new([], days: (ENV['NUM_DAYS'] || 7).to_i)
+      cli.invoke(:remove)
     end
 
     desc 'Set unknown attachment type for remote-only attachments'
@@ -548,21 +528,9 @@ namespace :mastodon do
     task clear: :environment do
       Pubsubhubbub::UnsubscribeWorker.push_bulk(Account.remote.without_followers.where.not(subscription_expires_at: nil).pluck(:id))
     end
-
-    desc 'Re-subscribes to soon expiring PuSH subscriptions (deprecated)'
-    task refresh: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
   end
 
   namespace :feeds do
-    desc 'Clear timelines of inactive users (deprecated)'
-    task clear: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-
     desc 'Clear all timelines without regenerating them'
     task clear_all: :environment do
       Redis.current.keys('feed:*').each { |key| Redis.current.del(key) }
@@ -576,21 +544,7 @@ namespace :mastodon do
     end
   end
 
-  namespace :emails do
-    desc 'Send out digest e-mails (deprecated)'
-    task digest: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-  end
-
   namespace :users do
-    desc 'Clear out unconfirmed users (deprecated)'
-    task clear: :environment do
-      # No-op
-      # This task is now executed via sidekiq-scheduler
-    end
-
     desc 'List e-mails of all admin users'
     task admins: :environment do
       puts 'Admin user emails:'
@@ -757,10 +711,10 @@ namespace :mastodon do
       pastel = Pastel.new
 
       duplicate_masters.each do |account|
-        puts pastel.yellow("First of their name: ") + pastel.bold(account.username) + " (#{admin_account_url(account.id)})"
+        puts pastel.yellow('First of their name: ') + pastel.bold(account.username) + " (#{admin_account_url(account.id)})"
 
         Account.where('lower(username) = ?', account.username.downcase).where.not(id: account.id).each do |duplicate|
-          puts "  " + pastel.red("Duplicate: ") + admin_account_url(duplicate.id)
+          puts '  ' + pastel.red('Duplicate: ') + admin_account_url(duplicate.id)
         end
       end
     end