From 55b210b3e599fa51186a646572056cda7aa3f23c Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 13 Dec 2022 20:02:32 +0100 Subject: Fix crash and incorrect behavior in tootctl domains crawl (#19004) --- lib/mastodon/domains_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index a7c78c4a7..77364ffbb 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -97,7 +97,7 @@ module Mastodon failed = Concurrent::AtomicFixnum.new(0) start_at = Time.now.to_f seed = start ? [start] : Instance.pluck(:domain) - blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$') + blocked_domains = /\.?(#{DomainBlock.where(severity: 1).pluck(:domain).map { |domain| Regexp.escape(domain) }.join('|')})$/ progress = create_progress_bar pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0) -- cgit From 78ef635980ff391ea3ba4c37de836947a97958a0 Mon Sep 17 00:00:00 2001 From: Evan <35814742+evanphilip@users.noreply.github.com> Date: Wed, 14 Dec 2022 19:50:07 +0100 Subject: Add command to remove avatar and header images of inactive remote accounts from the local database (#22149) * Add tootctl subcommand media remove-profile-media * Trigger workflows * Correcting external linting * External linting error * External linting fix * Merging with remove command * Linting * Correct long option names Co-authored-by: Claire * Correct long option names Co-authored-by: Claire * Correct long option names Co-authored-by: Claire * Remove saving a list of purged accounts Co-authored-by: Claire --- lib/mastodon/media_cli.rb | 78 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index bba4a1bd7..24cc98964 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -14,35 +14,78 @@ module Mastodon end option :days, type: :numeric, default: 7, aliases: [:d] + option :prune_profiles, type: :boolean, default: false + option :remove_headers, type: :boolean, default: false + option :include_follows, type: :boolean, default: false option :concurrency, type: :numeric, default: 5, aliases: [:c] - option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false - desc 'remove', 'Remove remote media files' + desc 'remove', 'Remove remote media files, headers or avatars' long_desc <<-DESC - Removes locally cached copies of media attachments from other servers. - + Removes locally cached copies of media attachments (and optionally profile + headers and avatars) from other servers. By default, only media attachements + are removed. The --days option specifies how old media attachments have to be before - they are removed. It defaults to 7 days. + they are removed. In case of avatars and headers, it specifies how old + the last webfinger request and update to the user has to be before they + are pruned. It defaults to 7 days. + If --prune-profiles is specified, only avatars and headers are removed. + If --remove-headers is specified, only headers are removed. + If --include-follows is specified along with --prune-profiles or + --remove-headers, all non-local profiles will be pruned irrespective of + follow status. By default, only accounts that are not followed by or + following anyone locally are pruned. DESC + # rubocop:disable Metrics/PerceivedComplexity def remove - time_ago = options[:days].days.ago - dry_run = options[:dry_run] ? '(DRY RUN)' : '' + if options[:prune_profiles] && options[:remove_headers] + say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true) + exit(1) + end + if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers]) + say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true) + exit(1) + end + time_ago = options[:days].days.ago + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' - processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| - next if media_attachment.file.blank? + if options[:prune_profiles] || options[:remove_headers] + processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account| + next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists? + next if account.avatar.blank? && account.header.blank? + next if options[:remove_headers] && account.header.blank? - size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) + size = (account.header_file_size || 0) + size += (account.avatar_file_size || 0) if options[:prune_profiles] - unless options[:dry_run] - media_attachment.file.destroy - media_attachment.thumbnail.destroy - media_attachment.save + unless options[:dry_run] + account.header.destroy + account.avatar.destroy if options[:prune_profiles] + account.save! + end + + size end - size + say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true) end - say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) + unless options[:prune_profiles] || options[:remove_headers] + processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment| + next if media_attachment.file.blank? + + size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) + + unless options[:dry_run] + media_attachment.file.destroy + media_attachment.thumbnail.destroy + media_attachment.save + end + + size + end + + say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) + end end option :start_after @@ -183,6 +226,7 @@ module Mastodon say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true) end + # rubocop:enable Metrics/PerceivedComplexity option :account, type: :string option :domain, type: :string @@ -269,7 +313,7 @@ module Mastodon def lookup(url) path = Addressable::URI.parse(url).path - path_segments = path.split('/')[2..-1] + path_segments = path.split('/')[2..] path_segments.delete('cache') unless [7, 10].include?(path_segments.size) -- cgit From f239d31f23b8bd55fb26f67906b815e4abe65d92 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 15 Dec 2022 14:52:50 +0100 Subject: Add --email and --dry-run options to `tootctl accounts delete` (#22328) --- lib/mastodon/accounts_cli.rb | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 77cbef84e..0dd852131 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -200,21 +200,44 @@ module Mastodon end end - desc 'delete USERNAME', 'Delete a user' + option :email + option :dry_run, type: :boolean + desc 'delete [USERNAME]', 'Delete a user' long_desc <<-LONG_DESC Remove a user account with a given USERNAME. - LONG_DESC - def delete(username) - account = Account.find_local(username) - if account.nil? - say('No user with such username', :red) + With the --email option, the user is selected based on email + rather than username. + LONG_DESC + def delete(username = nil) + if username.present? && options[:email].present? + say('Use username or --email, not both', :red) + exit(1) + elsif username.blank? && options[:email].blank? + say('No username provided', :red) exit(1) end - say("Deleting user with #{account.statuses_count} statuses, this might take a while...") - DeleteAccountService.new.call(account, reserve_email: false) - say('OK', :green) + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + account = nil + + if username.present? + account = Account.find_local(username) + if account.nil? + say('No user with such username', :red) + exit(1) + end + else + account = Account.left_joins(:user).find_by(user: { email: options[:email] }) + if account.nil? + say('No user with such email', :red) + exit(1) + end + end + + say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run}") + DeleteAccountService.new.call(account, reserve_email: false) unless options[:dry_run] + say("OK#{dry_run}", :green) end option :force, type: :boolean, aliases: [:f], description: 'Override public key check' -- cgit From 3d3429243fa0bacb20aac3db6c377441c0510f22 Mon Sep 17 00:00:00 2001 From: Dan Peterson Date: Thu, 15 Dec 2022 11:38:51 -0400 Subject: Fix default S3_HOSTNAME used in mastodon:setup (#19932) s3-us-east-1.amazonaws.com does not exist. Co-authored-by: Effy Elden --- lib/tasks/mastodon.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index c1e5bd2b4..3c891a07f 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -194,7 +194,7 @@ namespace :mastodon do env['S3_HOSTNAME'] = prompt.ask('S3 hostname:') do |q| q.required true - q.default 's3-us-east-1.amazonaws.com' + q.default 's3.us-east-1.amazonaws.com' q.modify :strip end -- cgit From 558ac411c469a19e9b32f78d4e4f7f29d46afa5f Mon Sep 17 00:00:00 2001 From: Nick Schonning Date: Thu, 5 Jan 2023 07:42:13 -0500 Subject: Expand Stylelint glob to include CSS files (#22469) --- lib/assets/wordmark.dark.css | 2 +- package.json | 2 +- public/inert.css | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/assets/wordmark.dark.css b/lib/assets/wordmark.dark.css index d87069178..db3355f59 100644 --- a/lib/assets/wordmark.dark.css +++ b/lib/assets/wordmark.dark.css @@ -1 +1 @@ -// Not needed +/* Not needed */ diff --git a/package.json b/package.json index 19e9018e8..291bd416c 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "test": "${npm_execpath} run test:lint:js && ${npm_execpath} run test:jest", "test:lint": "${npm_execpath} run test:lint:js && ${npm_execpath} run test:lint:sass", "test:lint:js": "eslint --ext=js . --cache", - "test:lint:sass": "stylelint '**/*.scss'", + "test:lint:sass": "stylelint \"**/*.{css,scss}\"", "test:jest": "cross-env NODE_ENV=test jest", "format": "prettier --write \"**/*.{json,yml}\"", "format-check": "prettier --check \"**/*.{json,yml}\"" diff --git a/public/inert.css b/public/inert.css index 275fad2e6..54e10616d 100644 --- a/public/inert.css +++ b/public/inert.css @@ -3,7 +3,8 @@ cursor: default; } -[inert], [inert] * { +[inert], +[inert] * { user-select: none; -webkit-user-select: none; -moz-user-select: none; -- cgit From a65f86ae5596d9c51a76cb05a3ebf5cd965df6ef Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 11 Jan 2023 21:53:11 +0100 Subject: Fix `$` not being escaped in `.env.production` file generated by `mastodon:setup` (#23012) * Fix `$` not being escaped in `.env.production` file generated by `mastodon:setup` * Improve robustness of dotenv escaping --- lib/tasks/mastodon.rake | 61 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 3c891a07f..cd6d1bcab 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -395,18 +395,11 @@ namespace :mastodon do incompatible_syntax = false env_contents = env.each_pair.map do |key, value| - if value.is_a?(String) && value =~ /[\s\#\\"]/ - incompatible_syntax = true + value = value.to_s + escaped = dotenv_escape(value) + incompatible_syntax = true if value != escaped - if value =~ /[']/ - value = value.to_s.gsub(/[\\"\$]/) { |x| "\\#{x}" } - "#{key}=\"#{value}\"" - else - "#{key}='#{value}'" - end - else - "#{key}=#{value}" - end + escaped end.join("\n") generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup @@ -519,3 +512,49 @@ def disable_log_stdout! HttpLog.configuration.logger = dev_null Paperclip.options[:log] = false end + +def dotenv_escape(value) + # Dotenv has its own parser, which unfortunately deviates somewhat from + # what shells actually do. + # + # In particular, we can't use Shellwords::escape because it outputs a + # non-quotable string, while Dotenv requires `#` to always be in quoted + # strings. + # + # Therefore, we need to write our own escape code… + # Dotenv's parser has a *lot* of edge cases, and I think not every + # ASCII string can even be represented into something Dotenv can parse, + # so this is a best effort thing. + # + # In particular, strings with all the following probably cannot be + # escaped: + # - `#`, or ends with spaces, which requires some form of quoting (simply escaping won't work) + # - `'` (single quote), preventing us from single-quoting + # - `\` followed by either `r` or `n` + + # No character that would cause Dotenv trouble + return value unless /[\s\#\\"'$]/.match?(value) + + # As long as the value doesn't include single quotes, we can safely + # rely on single quotes + return "'#{value}'" unless /[']/.match?(value) + + # If the value contains the string '\n' or '\r' we simply can't use + # a double-quoted string, because Dotenv will expand \n or \r no + # matter how much escaping we add. + double_quoting_disallowed = /\\[rn]/.match?(value) + + value = value.gsub(double_quoting_disallowed ? /[\\"'\s]/ : /[\\"']/) { |x| "\\#{x}" } + + # Dotenv is especially tricky with `$` as unbalanced + # parenthesis will make it not unescape `\$` as `$`… + + # Variables + value = value.gsub(/\$(?!\()/) { |x| "\\#{x}" } + # Commands + value = value.gsub(/\$(?\((?:[^()]|\g)+\))/) { |x| "\\#{x}" } + + value = "\"#{value}\"" unless double_quoting_disallowed + + value +end -- cgit From 15b88a83ab7dbe022e33552f45f300fc69a934d0 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 11 Jan 2023 22:21:10 +0100 Subject: Fix sanitizer parsing link text as HTML when stripping unsupported links (#22558) --- lib/sanitize_ext/sanitize_config.rb | 2 +- spec/lib/sanitize_config_spec.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb index a2e1d9d01..baf652662 100644 --- a/lib/sanitize_ext/sanitize_config.rb +++ b/lib/sanitize_ext/sanitize_config.rb @@ -49,7 +49,7 @@ class Sanitize end end - current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) + current_node.replace(Nokogiri::XML::Text.new(current_node.text, current_node.document)) unless LINK_PROTOCOLS.include?(scheme) end UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env| diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb index 747d81158..c9543ceb0 100644 --- a/spec/lib/sanitize_config_spec.rb +++ b/spec/lib/sanitize_config_spec.rb @@ -38,6 +38,10 @@ describe Sanitize::Config do expect(Sanitize.fragment('Test', subject)).to eq 'Test' end + it 'does not re-interpret HTML when removing unsupported links' do + expect(Sanitize.fragment('Test<a href="https://example.com">test</a>', subject)).to eq 'Test<a href="https://example.com">test</a>' + end + it 'keeps a with href' do expect(Sanitize.fragment('Test', subject)).to eq 'Test' end -- cgit From a3a5aa159783c7361771a32b97030f05fb40e574 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 13 Jan 2023 10:17:07 +0100 Subject: Fix incorrect env file generation in mastodon:setup (#23072) Regression from #23012 --- lib/tasks/mastodon.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index cd6d1bcab..32040feec 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -399,7 +399,7 @@ namespace :mastodon do escaped = dotenv_escape(value) incompatible_syntax = true if value != escaped - escaped + "#{key}=#{escaped}" end.join("\n") generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup -- cgit From 745bdb11a0d81cc4aff3fe3bba5eecdb8671a632 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 13 Jan 2023 17:00:23 +0100 Subject: Add `tootctl accounts migrate` (#22330) * Add tootctl accounts replay-migration Fixes #22281 * Change `tootctl accounts replay-migration` to `tootctl accounts migrate` --- lib/mastodon/accounts_cli.rb | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 0dd852131..693c9547c 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -553,6 +553,79 @@ module Mastodon end end + option :force, type: :boolean + option :replay, type: :boolean + option :target + desc 'migrate USERNAME', 'Migrate a local user to another account' + long_desc <<~LONG_DESC + With --replay, replay the last migration of the specified account, in + case some remote server may not have properly processed the associated + `Move` activity. + + With --target, specify another account to migrate to. + + With --force, perform the migration even if the selected account + redirects to a different account that the one specified. + LONG_DESC + def migrate(username) + if options[:replay].present? && options[:target].present? + say('Use --replay or --target, not both', :red) + exit(1) + end + + if options[:replay].blank? && options[:target].blank? + say('Use either --replay or --target', :red) + exit(1) + end + + account = Account.find_local(username) + + if account.nil? + say("No such account: #{username}", :red) + exit(1) + end + + migration = nil + + if options[:replay] + migration = account.migrations.last + if migration.nil? + say('The specified account has not performed any migration', :red) + exit(1) + end + + unless options[:force] || migration.target_acount_id == account.moved_to_account_id + say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red) + exit(1) + end + end + + if options[:target] + target_account = ResolveAccountService.new.call(options[:target]) + + if target_account.nil? + say("The specified target account could not be found: #{options[:target]}", :red) + exit(1) + end + + unless options[:force] || account.moved_to_account_id.nil? || account.moved_to_account_id == target_account.id + say('The specified account is redirecting to a different target account. Use --force if you want to change the migration target', :red) + exit(1) + end + + begin + migration = account.migrations.create!(acct: target_account.acct) + rescue ActiveRecord::RecordInvalid => e + say("Error: #{e.message}", :red) + exit(1) + end + end + + MoveService.new.call(migration) + + say("OK, migrated #{account.acct} to #{migration.target_account.acct}", :green) + end + private def rotate_keys_for_account(account, delay = 0) -- cgit From 0e8f8a1a1c225272596b3256e3adb0a20a0dc483 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Sat, 14 Jan 2023 06:34:16 +0900 Subject: Implement tootctl accounts prune (#18397) * Implement tootctl accounts prune * Optimise query Co-authored-by: Claire --- lib/mastodon/accounts_cli.rb | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 693c9547c..34afbc699 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -553,6 +553,43 @@ module Mastodon end end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :dry_run, type: :boolean + desc 'prune', 'Prune remote accounts that never interacted with local users' + long_desc <<-LONG_DESC + Prune remote account that + - follows no local accounts + - is not followed by any local accounts + - has no statuses on local + - has not been mentioned + - has not been favourited local posts + - not muted/blocked by us + LONG_DESC + def prune + dry_run = options[:dry_run] ? ' (dry run)' : '' + + query = Account.remote.where.not(actor_type: %i(Application Service)) + query = query.where('NOT EXISTS (SELECT 1 FROM mentions WHERE account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM favourites WHERE account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM statuses WHERE account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM follows WHERE account_id = accounts.id OR target_account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM blocks WHERE account_id = accounts.id OR target_account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM mutes WHERE target_account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM reports WHERE target_account_id = accounts.id)') + query = query.where('NOT EXISTS (SELECT 1 FROM follow_requests WHERE account_id = accounts.id OR target_account_id = accounts.id)') + + _, deleted = parallelize_with_progress(query) do |account| + next if account.bot? || account.group? + next if account.suspended? + next if account.silenced? + + account.destroy unless options[:dry_run] + 1 + end + + say("OK, pruned #{deleted} accounts#{dry_run}", :green) + end + option :force, type: :boolean option :replay, type: :boolean option :target -- cgit