Merge remote-tracking branch 'glitch/main'

6 files changed, 257 insertions, 41 deletions

diff --git a/lib/assets/wordmark.dark.css b/lib/assets/wordmark.dark.css
index d87069178..db3355f59 100644
--- a/lib/assets/wordmark.dark.css
+++ b/lib/assets/wordmark.dark.css
@@ -1 +1 @@
-// Not needed
+/* Not needed */
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
index 77cbef84e..34afbc699 100644
--- a/lib/mastodon/accounts_cli.rb
+++ b/lib/mastodon/accounts_cli.rb
@@ -200,21 +200,44 @@ module Mastodon
       end
     end
 
-    desc 'delete USERNAME', 'Delete a user'
+    option :email
+    option :dry_run, type: :boolean
+    desc 'delete [USERNAME]', 'Delete a user'
     long_desc <<-LONG_DESC
       Remove a user account with a given USERNAME.
-    LONG_DESC
-    def delete(username)
-      account = Account.find_local(username)
 
-      if account.nil?
-        say('No user with such username', :red)
+      With the --email option, the user is selected based on email
+      rather than username.
+    LONG_DESC
+    def delete(username = nil)
+      if username.present? && options[:email].present?
+        say('Use username or --email, not both', :red)
+        exit(1)
+      elsif username.blank? && options[:email].blank?
+        say('No username provided', :red)
         exit(1)
       end
 
-      say("Deleting user with #{account.statuses_count} statuses, this might take a while...")
-      DeleteAccountService.new.call(account, reserve_email: false)
-      say('OK', :green)
+      dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
+      account = nil
+
+      if username.present?
+        account = Account.find_local(username)
+        if account.nil?
+          say('No user with such username', :red)
+          exit(1)
+        end
+      else
+        account = Account.left_joins(:user).find_by(user: { email: options[:email] })
+        if account.nil?
+          say('No user with such email', :red)
+          exit(1)
+        end
+      end
+
+      say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run}")
+      DeleteAccountService.new.call(account, reserve_email: false) unless options[:dry_run]
+      say("OK#{dry_run}", :green)
     end
 
     option :force, type: :boolean, aliases: [:f], description: 'Override public key check'
@@ -530,6 +553,116 @@ module Mastodon
       end
     end
 
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :dry_run, type: :boolean
+    desc 'prune', 'Prune remote accounts that never interacted with local users'
+    long_desc <<-LONG_DESC
+      Prune remote account that
+      - follows no local accounts
+      - is not followed by any local accounts
+      - has no statuses on local
+      - has not been mentioned
+      - has not been favourited local posts
+      - not muted/blocked by us
+    LONG_DESC
+    def prune
+      dry_run = options[:dry_run] ? ' (dry run)' : ''
+
+      query = Account.remote.where.not(actor_type: %i(Application Service))
+      query = query.where('NOT EXISTS (SELECT 1 FROM mentions WHERE account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM favourites WHERE account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM statuses WHERE account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM follows WHERE account_id = accounts.id OR target_account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM blocks WHERE account_id = accounts.id OR target_account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM mutes WHERE target_account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM reports WHERE target_account_id = accounts.id)')
+      query = query.where('NOT EXISTS (SELECT 1 FROM follow_requests WHERE account_id = accounts.id OR target_account_id = accounts.id)')
+
+      _, deleted = parallelize_with_progress(query) do |account|
+        next if account.bot? || account.group?
+        next if account.suspended?
+        next if account.silenced?
+
+        account.destroy unless options[:dry_run]
+        1
+      end
+
+      say("OK, pruned #{deleted} accounts#{dry_run}", :green)
+    end
+
+    option :force, type: :boolean
+    option :replay, type: :boolean
+    option :target
+    desc 'migrate USERNAME', 'Migrate a local user to another account'
+    long_desc <<~LONG_DESC
+      With --replay, replay the last migration of the specified account, in
+      case some remote server may not have properly processed the associated
+      `Move` activity.
+
+      With --target, specify another account to migrate to.
+
+      With --force, perform the migration even if the selected account
+      redirects to a different account that the one specified.
+    LONG_DESC
+    def migrate(username)
+      if options[:replay].present? && options[:target].present?
+        say('Use --replay or --target, not both', :red)
+        exit(1)
+      end
+
+      if options[:replay].blank? && options[:target].blank?
+        say('Use either --replay or --target', :red)
+        exit(1)
+      end
+
+      account = Account.find_local(username)
+
+      if account.nil?
+        say("No such account: #{username}", :red)
+        exit(1)
+      end
+
+      migration = nil
+
+      if options[:replay]
+        migration = account.migrations.last
+        if migration.nil?
+          say('The specified account has not performed any migration', :red)
+          exit(1)
+        end
+
+        unless options[:force] || migration.target_acount_id == account.moved_to_account_id
+          say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red)
+          exit(1)
+        end
+      end
+
+      if options[:target]
+        target_account = ResolveAccountService.new.call(options[:target])
+
+        if target_account.nil?
+          say("The specified target account could not be found: #{options[:target]}", :red)
+          exit(1)
+        end
+
+        unless options[:force] || account.moved_to_account_id.nil? || account.moved_to_account_id == target_account.id
+          say('The specified account is redirecting to a different target account. Use --force if you want to change the migration target', :red)
+          exit(1)
+        end
+
+        begin
+          migration = account.migrations.create!(acct: target_account.acct)
+        rescue ActiveRecord::RecordInvalid => e
+          say("Error: #{e.message}", :red)
+          exit(1)
+        end
+      end
+
+      MoveService.new.call(migration)
+
+      say("OK, migrated #{account.acct} to #{migration.target_account.acct}", :green)
+    end
+
     private
 
     def rotate_keys_for_account(account, delay = 0)
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index a7c78c4a7..77364ffbb 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -97,7 +97,7 @@ module Mastodon
       failed          = Concurrent::AtomicFixnum.new(0)
       start_at        = Time.now.to_f
       seed            = start ? [start] : Instance.pluck(:domain)
-      blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$')
+      blocked_domains = /\.?(#{DomainBlock.where(severity: 1).pluck(:domain).map { |domain| Regexp.escape(domain) }.join('|')})$/
       progress        = create_progress_bar
 
       pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
index bba4a1bd7..24cc98964 100644
--- a/lib/mastodon/media_cli.rb
+++ b/lib/mastodon/media_cli.rb
@@ -14,35 +14,78 @@ module Mastodon
     end
 
     option :days, type: :numeric, default: 7, aliases: [:d]
+    option :prune_profiles, type: :boolean, default: false
+    option :remove_headers, type: :boolean, default: false
+    option :include_follows, type: :boolean, default: false
     option :concurrency, type: :numeric, default: 5, aliases: [:c]
-    option :verbose, type: :boolean, default: false, aliases: [:v]
     option :dry_run, type: :boolean, default: false
-    desc 'remove', 'Remove remote media files'
+    desc 'remove', 'Remove remote media files, headers or avatars'
     long_desc <<-DESC
-      Removes locally cached copies of media attachments from other servers.
-
+      Removes locally cached copies of media attachments (and optionally profile
+      headers and avatars) from other servers. By default, only media attachements
+      are removed.
       The --days option specifies how old media attachments have to be before
-      they are removed. It defaults to 7 days.
+      they are removed. In case of avatars and headers, it specifies how old
+      the last webfinger request and update to the user has to be before they
+      are pruned. It defaults to 7 days.
+      If --prune-profiles is specified, only avatars and headers are removed.
+      If --remove-headers is specified, only headers are removed.
+      If --include-follows is specified along with --prune-profiles or
+      --remove-headers, all non-local profiles will be pruned irrespective of
+      follow status. By default, only accounts that are not followed by or
+      following anyone locally are pruned.
     DESC
+    # rubocop:disable Metrics/PerceivedComplexity
     def remove
-      time_ago = options[:days].days.ago
-      dry_run  = options[:dry_run] ? '(DRY RUN)' : ''
+      if options[:prune_profiles] && options[:remove_headers]
+        say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true)
+        exit(1)
+      end
+      if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers])
+        say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true)
+        exit(1)
+      end
+      time_ago        = options[:days].days.ago
+      dry_run         = options[:dry_run] ? ' (DRY RUN)' : ''
 
-      processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment|
-        next if media_attachment.file.blank?
+      if options[:prune_profiles] || options[:remove_headers]
+        processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account|
+          next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists?
+          next if account.avatar.blank? && account.header.blank?
+          next if options[:remove_headers] && account.header.blank?
 
-        size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
+          size = (account.header_file_size || 0)
+          size += (account.avatar_file_size || 0) if options[:prune_profiles]
 
-        unless options[:dry_run]
-          media_attachment.file.destroy
-          media_attachment.thumbnail.destroy
-          media_attachment.save
+          unless options[:dry_run]
+            account.header.destroy
+            account.avatar.destroy if options[:prune_profiles]
+            account.save!
+          end
+
+          size
         end
 
-        size
+        say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true)
       end
 
-      say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true)
+      unless options[:prune_profiles] || options[:remove_headers]
+        processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment|
+          next if media_attachment.file.blank?
+
+          size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
+
+          unless options[:dry_run]
+            media_attachment.file.destroy
+            media_attachment.thumbnail.destroy
+            media_attachment.save
+          end
+
+          size
+        end
+
+        say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
+      end
     end
 
     option :start_after
@@ -183,6 +226,7 @@ module Mastodon
 
       say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
     end
+    # rubocop:enable Metrics/PerceivedComplexity
 
     option :account, type: :string
     option :domain, type: :string
@@ -269,7 +313,7 @@ module Mastodon
     def lookup(url)
       path = Addressable::URI.parse(url).path
 
-      path_segments = path.split('/')[2..-1]
+      path_segments = path.split('/')[2..]
       path_segments.delete('cache')
 
       unless [7, 10].include?(path_segments.size)
diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb
index 27e1eb92f..55eb17097 100644
--- a/lib/sanitize_ext/sanitize_config.rb
+++ b/lib/sanitize_ext/sanitize_config.rb
@@ -69,7 +69,7 @@ class Sanitize
         end
       end
 
-      current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme)
+      current_node.replace(Nokogiri::XML::Text.new(current_node.text, current_node.document)) unless LINK_PROTOCOLS.include?(scheme)
     end
 
     MASTODON_STRICT ||= freeze_config(
diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index c1e5bd2b4..32040feec 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -194,7 +194,7 @@ namespace :mastodon do
 
           env['S3_HOSTNAME'] = prompt.ask('S3 hostname:') do |q|
             q.required true
-            q.default 's3-us-east-1.amazonaws.com'
+            q.default 's3.us-east-1.amazonaws.com'
             q.modify :strip
           end
 
@@ -395,18 +395,11 @@ namespace :mastodon do
         incompatible_syntax = false
 
         env_contents = env.each_pair.map do |key, value|
-          if value.is_a?(String) && value =~ /[\s\#\\"]/
-            incompatible_syntax = true
+          value = value.to_s
+          escaped = dotenv_escape(value)
+          incompatible_syntax = true if value != escaped
 
-            if value =~ /[']/
-              value = value.to_s.gsub(/[\\"\$]/) { |x| "\\#{x}" }
-              "#{key}=\"#{value}\""
-            else
-              "#{key}='#{value}'"
-            end
-          else
-            "#{key}=#{value}"
-          end
+          "#{key}=#{escaped}"
         end.join("\n")
 
         generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup
@@ -519,3 +512,49 @@ def disable_log_stdout!
   HttpLog.configuration.logger = dev_null
   Paperclip.options[:log]      = false
 end
+
+def dotenv_escape(value)
+  # Dotenv has its own parser, which unfortunately deviates somewhat from
+  # what shells actually do.
+  #
+  # In particular, we can't use Shellwords::escape because it outputs a
+  # non-quotable string, while Dotenv requires `#` to always be in quoted
+  # strings.
+  #
+  # Therefore, we need to write our own escape code…
+  # Dotenv's parser has a *lot* of edge cases, and I think not every
+  # ASCII string can even be represented into something Dotenv can parse,
+  # so this is a best effort thing.
+  #
+  # In particular, strings with all the following probably cannot be
+  # escaped:
+  # - `#`, or ends with spaces, which requires some form of quoting (simply escaping won't work)
+  # - `'` (single quote), preventing us from single-quoting
+  # - `\` followed by either `r` or `n`
+
+  # No character that would cause Dotenv trouble
+  return value unless /[\s\#\\"'$]/.match?(value)
+
+  # As long as the value doesn't include single quotes, we can safely
+  # rely on single quotes
+  return "'#{value}'" unless /[']/.match?(value)
+
+  # If the value contains the string '\n' or '\r' we simply can't use
+  # a double-quoted string, because Dotenv will expand \n or \r no
+  # matter how much escaping we add.
+  double_quoting_disallowed = /\\[rn]/.match?(value)
+
+  value = value.gsub(double_quoting_disallowed ? /[\\"'\s]/ : /[\\"']/) { |x| "\\#{x}" }
+
+  # Dotenv is especially tricky with `$` as unbalanced
+  # parenthesis will make it not unescape `\$` as `$`…
+
+  # Variables
+  value = value.gsub(/\$(?!\()/) { |x| "\\#{x}" }
+  # Commands
+  value = value.gsub(/\$(?<cmd>\((?:[^()]|\g<cmd>)+\))/) { |x| "\\#{x}" }
+
+  value = "\"#{value}\"" unless double_quoting_disallowed
+
+  value
+end