diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/cli.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/domains_cli.rb | 8 | ||||
| -rw-r--r-- | lib/mastodon/email_domain_blocks_cli.rb | 138 | ||||
| -rw-r--r-- | lib/mastodon/media_cli.rb | 10 | ||||
| -rw-r--r-- | lib/paperclip/attachment_extensions.rb | 2 | ||||
| -rw-r--r-- | lib/paperclip/image_extractor.rb | 49 | ||||
| -rw-r--r-- | lib/paperclip/type_corrector.rb | 10 |
7 files changed, 208 insertions, 13 deletions
diff --git a/lib/cli.rb b/lib/cli.rb index 313a36a3d..9162144cc 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -12,6 +12,7 @@ require_relative 'mastodon/domains_cli' require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/upgrade_cli' +require_relative 'mastodon/email_domain_blocks_cli' require_relative 'mastodon/version' module Mastodon @@ -53,6 +54,9 @@ module Mastodon desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities' subcommand 'upgrade', Mastodon::UpgradeCLI + desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks' + subcommand 'email_domain_blocks', Mastodon::EmailDomainBlocksCLI + option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index b5435bb5e..558737c27 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -16,22 +16,22 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean - option :whitelist_mode, type: :boolean + option :limited_federation_mode, type: :boolean desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any records. Unlike a suspension, if the DOMAIN still exists in the wild, it means the accounts could return if they are resolved again. - When the --whitelist-mode option is given, instead of purging accounts - from a single domain, all accounts from domains that are not whitelisted + When the --limited-federation-mode option is given, instead of purging accounts + from a single domain, all accounts from domains that have not been explicitly allowed are removed from the database. LONG_DESC def purge(*domains) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' scope = begin - if options[:whitelist_mode] + if options[:limited_federation_mode] Account.remote.where.not(domain: DomainAllow.pluck(:domain)) elsif !domains.empty? Account.remote.where(domain: domains) diff --git a/lib/mastodon/email_domain_blocks_cli.rb b/lib/mastodon/email_domain_blocks_cli.rb new file mode 100644 index 000000000..7fe1efaaa --- /dev/null +++ b/lib/mastodon/email_domain_blocks_cli.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'concurrent' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class EmailDomainBlocksCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + desc 'list', 'List blocked e-mail domains' + def list + EmailDomainBlock.where(parent_id: nil).order(id: 'DESC').find_each do |entry| + say(entry.domain.to_s, :white) + + EmailDomainBlock.where(parent_id: entry.id).order(id: 'DESC').find_each do |child| + say(" #{child.domain}", :cyan) + end + end + end + + option :with_dns_records, type: :boolean + desc 'add DOMAIN...', 'Block e-mail domain(s)' + long_desc <<-LONG_DESC + Blocking an e-mail domain prevents users from signing up + with e-mail addresses from that domain. You can provide one or + multiple domains to the command. + + When the --with-dns-records option is given, an attempt to resolve the + given domains' DNS records will be made and the results (A, AAAA and MX) will + also be blocked. This can be helpful if you are blocking an e-mail server that + has many different domains pointing to it as it allows you to essentially block + it at the root. + LONG_DESC + def add(*domains) + if domains.empty? + say('No domain(s) given', :red) + exit(1) + end + + skipped = 0 + processed = 0 + + domains.each do |domain| + if EmailDomainBlock.where(domain: domain).exists? + say("#{domain} is already blocked.", :yellow) + skipped += 1 + next + end + + email_domain_block = EmailDomainBlock.new(domain: domain, with_dns_records: options[:with_dns_records] || false) + email_domain_block.save! + processed += 1 + + next unless email_domain_block.with_dns_records? + + hostnames = [] + ips = [] + + Resolv::DNS.open do |dns| + dns.timeouts = 1 + hostnames = dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s } + + ([email_domain_block.domain] + hostnames).uniq.each do |hostname| + ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s }) + ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s }) + end + end + + (hostnames + ips).uniq.each do |hostname| + another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: email_domain_block) + + if EmailDomainBlock.where(domain: hostname).exists? + say("#{hostname} is already blocked.", :yellow) + skipped += 1 + next + end + + another_email_domain_block.save! + processed += 1 + end + end + + say("Added #{processed}, skipped #{skipped}", color(processed, 0)) + end + + desc 'remove DOMAIN...', 'Remove e-mail domain blocks' + def remove(*domains) + if domains.empty? + say('No domain(s) given', :red) + exit(1) + end + + skipped = 0 + processed = 0 + failed = 0 + + domains.each do |domain| + entry = EmailDomainBlock.find_by(domain: domain) + + if entry.nil? + say("#{domain} is not yet blocked.", :yellow) + skipped += 1 + next + end + + children_count = EmailDomainBlock.where(parent_id: entry.id).count + result = entry.destroy + + if result + processed += 1 + children_count + else + say("#{domain} could not be unblocked.", :red) + failed += 1 + end + end + + say("Removed #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed)) + end + + private + + def color(processed, failed) + if !processed.zero? && failed.zero? + :green + elsif failed.zero? + :yellow + else + :red + end + end + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index c95f3410a..2a4e3e379 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -31,10 +31,11 @@ module Mastodon processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| next if media_attachment.file.blank? - size = media_attachment.file_file_size + size = media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0) unless options[:dry_run] media_attachment.file.destroy + media_attachment.thumbnail.destroy media_attachment.save end @@ -227,11 +228,12 @@ module Mastodon next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) unless options[:dry_run] - media_attachment.file_remote_url = media_attachment.remote_url + media_attachment.reset_file! + media_attachment.reset_thumbnail! media_attachment.save end - media_attachment.file_file_size + media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0) end say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) @@ -239,7 +241,7 @@ module Mastodon desc 'usage', 'Calculate disk space consumed by Mastodon' def usage - say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(:file_file_size))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(:file_file_size))} local)") + say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} local)") say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)") say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}") say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)") diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index f3e51dbd3..93df0a326 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -7,7 +7,7 @@ module Paperclip # usage, and we still want to generate thumbnails straight # away, it's the only style we need to exclude def process_style?(style_name, style_args) - if style_name == :original && instance.respond_to?(:delay_processing?) && instance.delay_processing? + if style_name == :original && instance.respond_to?(:delay_processing_for_attachment?) && instance.delay_processing_for_attachment?(name) false else style_args.empty? || style_args.include?(style_name) diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb new file mode 100644 index 000000000..114852e8b --- /dev/null +++ b/lib/paperclip/image_extractor.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'mime/types/columnar' + +module Paperclip + class ImageExtractor < Paperclip::Processor + IMAGE_EXTRACTION_OPTIONS = { + convert_options: { + output: { + 'loglevel' => 'fatal', + vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease', + }.freeze, + }.freeze, + format: 'png', + time: -1, + file_geometry_parser: FastGeometryParser, + }.freeze + + def make + return @file unless options[:style] == :original + + image = begin + begin + Paperclip::Transcoder.make(file, IMAGE_EXTRACTION_OPTIONS.dup, attachment) + rescue Paperclip::Error, ::Av::CommandError + nil + end + end + + unless image.nil? + begin + attachment.instance.thumbnail = image if image.size.positive? + ensure + # Paperclip does not automatically delete the source file of + # a new attachment while working on copies of it, so we need + # to make sure it's cleaned up + + begin + FileUtils.rm(image) + rescue Errno::ENOENT + nil + end + end + end + + @file + end + end +end diff --git a/lib/paperclip/type_corrector.rb b/lib/paperclip/type_corrector.rb index 0b0c10a56..17e2fc5da 100644 --- a/lib/paperclip/type_corrector.rb +++ b/lib/paperclip/type_corrector.rb @@ -5,13 +5,15 @@ require 'mime/types/columnar' module Paperclip class TypeCorrector < Paperclip::Processor def make - target_extension = options[:format] - extension = File.extname(attachment.instance.file_file_name) + return @file unless options[:format] + + target_extension = '.' + options[:format] + extension = File.extname(attachment.instance_read(:file_name)) return @file unless options[:style] == :original && target_extension && extension != target_extension - attachment.instance.file_content_type = options[:content_type] || attachment.instance.file_content_type - attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.' + target_extension + attachment.instance_write(:content_type, options[:content_type] || attachment.instance_read(:content_type)) + attachment.instance_write(:file_name, File.basename(attachment.instance_read(:file_name), '.*') + target_extension) @file end |