diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/cli.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/cli_helper.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/emoji_cli.rb | 44 | ||||
| -rw-r--r-- | lib/mastodon/media_cli.rb | 24 | ||||
| -rw-r--r-- | lib/mastodon/upgrade_cli.rb | 148 | ||||
| -rw-r--r-- | lib/paperclip/attachment_extensions.rb | 16 | ||||
| -rw-r--r-- | lib/tasks/emojis.rake | 2 |
7 files changed, 224 insertions, 18 deletions
diff --git a/lib/cli.rb b/lib/cli.rb index 19cc5d6b5..313a36a3d 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -11,6 +11,7 @@ require_relative 'mastodon/statuses_cli' require_relative 'mastodon/domains_cli' require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' +require_relative 'mastodon/upgrade_cli' require_relative 'mastodon/version' module Mastodon @@ -49,6 +50,9 @@ module Mastodon desc 'cache SUBCOMMAND ...ARGS', 'Manage cache' subcommand 'cache', Mastodon::CacheCLI + desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities' + subcommand 'upgrade', Mastodon::UpgradeCLI + option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index ec4d9a81e..4a20fa8d6 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -10,6 +10,10 @@ Paperclip.options[:log] = false module Mastodon module CLIHelper + def dry_run? + options[:dry_run] + end + def create_progress_bar(total = nil) ProgressBar.create(total: total, format: '%c/%u |%b%i| %e') end diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index dbaf12018..da8fd6a0d 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -23,7 +23,7 @@ module Mastodon Existing emoji will be skipped unless the --overwrite option is provided, in which case they will be overwritten. - You can specifiy a --category under which the emojis will be + You can specify a --category under which the emojis will be grouped together. With the --prefix option, a prefix can be added to all @@ -72,6 +72,48 @@ module Mastodon say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed)) end + option :category + option :overwrite, type: :boolean + desc 'export PATH', 'Export emoji to a TAR GZIP archive at PATH' + long_desc <<-LONG_DESC + Exports custom emoji to 'export.tar.gz' at PATH. + + The --category option dumps only the specified category. + If this option is not specified, all emoji will be exported. + + The --overwrite option will overwrite an existing archive. + LONG_DESC + def export(path) + exported = 0 + category = CustomEmojiCategory.find_by(name: options[:category]) + export_file_name = File.join(path, 'export.tar.gz') + + if File.file?(export_file_name) && !options[:overwrite] + say("Archive already exists! Use '--overwrite' to overwrite it!") + exit 1 + end + if category.nil? && options[:category] + say("Unable to find category '#{options[:category]}'!") + exit 1 + end + + File.open(export_file_name, 'wb') do |file| + Zlib::GzipWriter.wrap(file) do |gzip| + Gem::Package::TarWriter.new(gzip) do |tar| + scope = !options[:category] || category.nil? ? CustomEmoji.local : category.emojis + scope.find_each do |emoji| + say("Adding '#{emoji.shortcode}'...") + tar.add_file_simple(emoji.shortcode + File.extname(emoji.image_file_name), 0o644, emoji.image_file_size) do |io| + io.write Paperclip.io_adapters.for(emoji.image).read + exported += 1 + end + end + end + end + end + say("Exported #{exported}") + end + option :remote_only, type: :boolean desc 'purge', 'Remove all custom emoji' long_desc <<-LONG_DESC diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 08a8f1093..424d65a5f 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -85,7 +85,9 @@ module Mastodon record_map = preload_records_from_mixed_objects(objects) objects.each do |object| - path_segments = object.key.split('/') + path_segments = object.key.split('/') + path_segments.delete('cache') + model_name = path_segments.first.classify attachment_name = path_segments[1].singularize record_id = path_segments[2..-2].join.to_i @@ -120,8 +122,11 @@ module Mastodon Find.find(File.join(*[root_path, prefix].compact)) do |path| next if File.directory?(path) - key = path.gsub("#{root_path}#{File::SEPARATOR}", '') - path_segments = key.split(File::SEPARATOR) + key = path.gsub("#{root_path}#{File::SEPARATOR}", '') + + path_segments = key.split(File::SEPARATOR) + path_segments.delete('cache') + model_name = path_segments.first.classify record_id = path_segments[2..-2].join.to_i attachment_name = path_segments[1].singularize @@ -229,10 +234,13 @@ module Mastodon desc 'lookup URL', 'Lookup where media is displayed by passing a media URL' def lookup(url) - path = Addressable::URI.parse(url).path + path = Addressable::URI.parse(url).path + path_segments = path.split('/')[2..-1] - model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + path_segments.delete('cache') + + model_name = path_segments.first.classify + record_id = path_segments[2..-2].join.to_i unless PRELOAD_MODEL_WHITELIST.include?(model_name) say("Cannot find corresponding model: #{model_name}", :red) @@ -276,7 +284,9 @@ module Mastodon preload_map = Hash.new { |hash, key| hash[key] = [] } objects.map do |object| - segments = object.key.split('/').first + segments = object.key.split('/') + segments.delete('cache') + model_name = segments.first.classify record_id = segments[2..-2].join.to_i diff --git a/lib/mastodon/upgrade_cli.rb b/lib/mastodon/upgrade_cli.rb new file mode 100644 index 000000000..74d13f62d --- /dev/null +++ b/lib/mastodon/upgrade_cli.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class UpgradeCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + CURRENT_STORAGE_SCHEMA_VERSION = 1 + + option :dry_run, type: :boolean, default: false + option :verbose, type: :boolean, default: false, aliases: [:v] + desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version' + long_desc <<~LONG_DESC + Iterates over every file attachment of every record and, if its storage schema is outdated, performs the + necessary upgrade to the latest one. In practice this means e.g. moving files to different directories. + + Will most likely take a long time. + LONG_DESC + def storage_schema + progress = create_progress_bar(nil) + dry_run = dry_run? ? ' (DRY RUN)' : '' + records = 0 + + klasses = [ + Account, + CustomEmoji, + MediaAttachment, + PreviewCard, + ] + + klasses.each do |klass| + attachment_names = klass.attachment_definitions.keys + + klass.find_each do |record| + attachment_names.each do |attachment_name| + attachment = record.public_send(attachment_name) + + next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION + + attachment.styles.each_key do |style| + case Paperclip::Attachment.default_options[:storage] + when :s3 + upgrade_storage_s3(progress, attachment, style) + when :fog + upgrade_storage_fog(progress, attachment, style) + when :filesystem + upgrade_storage_filesystem(progress, attachment, style) + end + + progress.increment + end + + attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) + end + + if record.changed? + record.save unless dry_run? + records += 1 + end + end + end + + progress.total = progress.progress + progress.finish + + say("Upgraded storage schema of #{records} records#{dry_run}", :green, true) + end + + private + + def upgrade_storage_s3(progress, attachment, style) + previous_storage_schema_version = attachment.storage_schema_version + object = attachment.s3_object(style) + + attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) + + upgraded_path = attachment.path(style) + + if upgraded_path != object.key && object.exists? + progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose] + + begin + object.move_to(upgraded_path) unless dry_run? + rescue => e + progress.log(pastel.red("Error processing #{object.key}: #{e}")) + end + end + + # Because we move files style-by-style, it's important to restore + # previous version at the end. The upgrade will be recorded after + # all styles are updated + attachment.instance_write(:storage_schema_version, previous_storage_schema_version) + end + + def upgrade_storage_fog(_progress, _attachment, _style) + say('The fog storage driver is not supported for this operation at this time', :red) + exit(1) + end + + def upgrade_storage_filesystem(progress, attachment, style) + previous_storage_schema_version = attachment.storage_schema_version + previous_path = attachment.path(style) + + attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) + + upgraded_path = attachment.path(style) + + if upgraded_path != previous_path && File.exist?(previous_path) + progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose] + + begin + unless dry_run? + FileUtils.mkdir_p(File.dirname(upgraded_path)) + FileUtils.mv(previous_path, upgraded_path) + + begin + FileUtils.rmdir(previous_path, parents: true) + rescue Errno::ENOTEMPTY + # OK + end + end + rescue => e + progress.log(pastel.red("Error processing #{previous_path}: #{e}")) + + unless dry_run? + begin + FileUtils.rmdir(upgraded_path, parents: true) + rescue Errno::ENOTEMPTY + # OK + end + end + end + end + + # Because we move files style-by-style, it's important to restore + # previous version at the end. The upgrade will be recorded after + # all styles are updated + attachment.instance_write(:storage_schema_version, previous_storage_schema_version) + end + end +end diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index d9ec0159a..f3e51dbd3 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -14,15 +14,13 @@ module Paperclip end end - def reprocess_original! - old_original_path = path(:original) - reprocess!(:original) - new_original_path = path(:original) - - if new_original_path != old_original_path - @queued_for_delete << old_original_path - flush_deletes - end + def storage_schema_version + instance_read(:storage_schema_version) || 0 + end + + def assign_attributes + super + instance_write(:storage_schema_version, 1) end def variant?(other_filename) diff --git a/lib/tasks/emojis.rake b/lib/tasks/emojis.rake index 892afd898..70919fbdb 100644 --- a/lib/tasks/emojis.rake +++ b/lib/tasks/emojis.rake @@ -15,7 +15,7 @@ end namespace :emojis do desc 'Generate a unicode to filename mapping' task :generate do - source = 'http://www.unicode.org/Public/emoji/11.0/emoji-test.txt' + source = 'http://www.unicode.org/Public/emoji/12.0/emoji-test.txt' codes = [] dest = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json') |