Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- `app/controllers/statuses_controller.rb`:
  Upstream disabled the embed controller for reblogs.
  Not a real conflict, but glitch-soc has an extra line to deal
  with its theming system.
  Ported upstream changes.
- `app/javascript/packs/public.js`:
  Upstream made changes to get rid of most inline CSS, this changes
  javascript for public pages, which in glitch are split between
  different files. Ported those changes.
- `app/models/status.rb`:
  Upstream changed the block check in `Status#permitted_for` to
  include domain-block checks. Not a real conflict with glitch-soc,
  but our scope is slightly different, as our scope for
  unauthenticated access do not include instance-local toots.
  Ported upstream changes.
- `app/serializers/rest/instance_serializer.rb`:
  Not a real conflict, upstream added a new field to the instance
  serializer, the conflict is one line above since we added more of
  that.
  Ported upstream changes.
- `app/views/settings/profiles/show.html.haml`:
  Upstream got rid of most inline CSS and moved hidden elements
  to data attributes in the process, in fields were we have
  different values.
  Ported upstream changes while keeping our glitch-specific
  values.
- `app/views/statuses/_simple_status.html.haml`:
  Upstream got rid of inline CSS on an HAML line we treat
  differently, stripping empty text nodes.
  Ported upstream changes to the style attribute, keeping
  the empty text node stripping behavior.

6 files changed, 225 insertions, 8 deletions

diff --git a/lib/cli.rb b/lib/cli.rb
index 19cc5d6b5..313a36a3d 100644
--- a/lib/cli.rb
+++ b/lib/cli.rb
@@ -11,6 +11,7 @@ require_relative 'mastodon/statuses_cli'
 require_relative 'mastodon/domains_cli'
 require_relative 'mastodon/preview_cards_cli'
 require_relative 'mastodon/cache_cli'
+require_relative 'mastodon/upgrade_cli'
 require_relative 'mastodon/version'
 
 module Mastodon
@@ -49,6 +50,9 @@ module Mastodon
     desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
     subcommand 'cache', Mastodon::CacheCLI
 
+    desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
+    subcommand 'upgrade', Mastodon::UpgradeCLI
+
     option :dry_run, type: :boolean
     desc 'self-destruct', 'Erase the server from the federation'
     long_desc <<~LONG_DESC
diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb
index ec4d9a81e..4a20fa8d6 100644
--- a/lib/mastodon/cli_helper.rb
+++ b/lib/mastodon/cli_helper.rb
@@ -10,6 +10,10 @@ Paperclip.options[:log]      = false
 
 module Mastodon
   module CLIHelper
+    def dry_run?
+      options[:dry_run]
+    end
+
     def create_progress_bar(total = nil)
       ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
     end
diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb
index dbaf12018..da8fd6a0d 100644
--- a/lib/mastodon/emoji_cli.rb
+++ b/lib/mastodon/emoji_cli.rb
@@ -23,7 +23,7 @@ module Mastodon
       Existing emoji will be skipped unless the --overwrite option
       is provided, in which case they will be overwritten.
 
-      You can specifiy a --category under which the emojis will be
+      You can specify a --category under which the emojis will be
       grouped together.
 
       With the --prefix option, a prefix can be added to all
@@ -72,6 +72,48 @@ module Mastodon
       say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
     end
 
+    option :category
+    option :overwrite, type: :boolean
+    desc 'export PATH', 'Export emoji to a TAR GZIP archive at PATH'
+    long_desc <<-LONG_DESC
+      Exports custom emoji to 'export.tar.gz' at PATH.
+
+      The --category option dumps only the specified category.
+      If this option is not specified, all emoji will be exported.
+
+      The --overwrite option will overwrite an existing archive.
+    LONG_DESC
+    def export(path)
+      exported         = 0
+      category         = CustomEmojiCategory.find_by(name: options[:category])
+      export_file_name = File.join(path, 'export.tar.gz')
+
+      if File.file?(export_file_name) && !options[:overwrite]
+        say("Archive already exists! Use '--overwrite' to overwrite it!")
+        exit 1
+      end
+      if category.nil? && options[:category]
+        say("Unable to find category '#{options[:category]}'!")
+        exit 1
+      end
+
+      File.open(export_file_name, 'wb') do |file|
+        Zlib::GzipWriter.wrap(file) do |gzip|
+          Gem::Package::TarWriter.new(gzip) do |tar|
+            scope = !options[:category] || category.nil? ? CustomEmoji.local : category.emojis
+            scope.find_each do |emoji|
+              say("Adding '#{emoji.shortcode}'...")
+              tar.add_file_simple(emoji.shortcode + File.extname(emoji.image_file_name), 0o644, emoji.image_file_size) do |io|
+                io.write Paperclip.io_adapters.for(emoji.image).read
+                exported += 1
+              end
+            end
+          end
+        end
+      end
+      say("Exported #{exported}")
+    end
+
     option :remote_only, type: :boolean
     desc 'purge', 'Remove all custom emoji'
     long_desc <<-LONG_DESC
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
index 0f211f272..424d65a5f 100644
--- a/lib/mastodon/media_cli.rb
+++ b/lib/mastodon/media_cli.rb
@@ -85,7 +85,9 @@ module Mastodon
           record_map = preload_records_from_mixed_objects(objects)
 
           objects.each do |object|
-            path_segments   = object.key.split('/')
+            path_segments = object.key.split('/')
+            path_segments.delete('cache')
+
             model_name      = path_segments.first.classify
             attachment_name = path_segments[1].singularize
             record_id       = path_segments[2..-2].join.to_i
@@ -120,8 +122,11 @@ module Mastodon
         Find.find(File.join(*[root_path, prefix].compact)) do |path|
           next if File.directory?(path)
 
-          key             = path.gsub("#{root_path}#{File::SEPARATOR}", '')
-          path_segments   = key.split(File::SEPARATOR)
+          key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
+
+          path_segments = key.split(File::SEPARATOR)
+          path_segments.delete('cache')
+
           model_name      = path_segments.first.classify
           record_id       = path_segments[2..-2].join.to_i
           attachment_name = path_segments[1].singularize
@@ -229,10 +234,13 @@ module Mastodon
 
     desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
     def lookup(url)
-      path          = Addressable::URI.parse(url).path
+      path = Addressable::URI.parse(url).path
+
       path_segments = path.split('/')[2..-1]
-      model_name    = path_segments.first.classify
-      record_id     = path_segments[2..-2].join.to_i
+      path_segments.delete('cache')
+
+      model_name = path_segments.first.classify
+      record_id  = path_segments[2..-2].join.to_i
 
       unless PRELOAD_MODEL_WHITELIST.include?(model_name)
         say("Cannot find corresponding model: #{model_name}", :red)
@@ -276,7 +284,9 @@ module Mastodon
       preload_map = Hash.new { |hash, key| hash[key] = [] }
 
       objects.map do |object|
-        segments   = object.key.split('/')
+        segments = object.key.split('/')
+        segments.delete('cache')
+
         model_name = segments.first.classify
         record_id  = segments[2..-2].join.to_i
 
diff --git a/lib/mastodon/upgrade_cli.rb b/lib/mastodon/upgrade_cli.rb
new file mode 100644
index 000000000..74d13f62d
--- /dev/null
+++ b/lib/mastodon/upgrade_cli.rb
@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+module Mastodon
+  class UpgradeCLI < Thor
+    include CLIHelper
+
+    def self.exit_on_failure?
+      true
+    end
+
+    CURRENT_STORAGE_SCHEMA_VERSION = 1
+
+    option :dry_run, type: :boolean, default: false
+    option :verbose, type: :boolean, default: false, aliases: [:v]
+    desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
+    long_desc <<~LONG_DESC
+      Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
+      necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.
+
+      Will most likely take a long time.
+    LONG_DESC
+    def storage_schema
+      progress = create_progress_bar(nil)
+      dry_run  = dry_run? ? ' (DRY RUN)' : ''
+      records  = 0
+
+      klasses = [
+        Account,
+        CustomEmoji,
+        MediaAttachment,
+        PreviewCard,
+      ]
+
+      klasses.each do |klass|
+        attachment_names = klass.attachment_definitions.keys
+
+        klass.find_each do |record|
+          attachment_names.each do |attachment_name|
+            attachment = record.public_send(attachment_name)
+
+            next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION
+
+            attachment.styles.each_key do |style|
+              case Paperclip::Attachment.default_options[:storage]
+              when :s3
+                upgrade_storage_s3(progress, attachment, style)
+              when :fog
+                upgrade_storage_fog(progress, attachment, style)
+              when :filesystem
+                upgrade_storage_filesystem(progress, attachment, style)
+              end
+
+              progress.increment
+            end
+
+            attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
+          end
+
+          if record.changed?
+            record.save unless dry_run?
+            records += 1
+          end
+        end
+      end
+
+      progress.total = progress.progress
+      progress.finish
+
+      say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
+    end
+
+    private
+
+    def upgrade_storage_s3(progress, attachment, style)
+      previous_storage_schema_version = attachment.storage_schema_version
+      object                          = attachment.s3_object(style)
+
+      attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
+
+      upgraded_path = attachment.path(style)
+
+      if upgraded_path != object.key && object.exists?
+        progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose]
+
+        begin
+          object.move_to(upgraded_path) unless dry_run?
+        rescue => e
+          progress.log(pastel.red("Error processing #{object.key}: #{e}"))
+        end
+      end
+
+      # Because we move files style-by-style, it's important to restore
+      # previous version at the end. The upgrade will be recorded after
+      # all styles are updated
+      attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
+    end
+
+    def upgrade_storage_fog(_progress, _attachment, _style)
+      say('The fog storage driver is not supported for this operation at this time', :red)
+      exit(1)
+    end
+
+    def upgrade_storage_filesystem(progress, attachment, style)
+      previous_storage_schema_version = attachment.storage_schema_version
+      previous_path                   = attachment.path(style)
+
+      attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
+
+      upgraded_path = attachment.path(style)
+
+      if upgraded_path != previous_path && File.exist?(previous_path)
+        progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]
+
+        begin
+          unless dry_run?
+            FileUtils.mkdir_p(File.dirname(upgraded_path))
+            FileUtils.mv(previous_path, upgraded_path)
+
+            begin
+              FileUtils.rmdir(previous_path, parents: true)
+            rescue Errno::ENOTEMPTY
+              # OK
+            end
+          end
+        rescue => e
+          progress.log(pastel.red("Error processing #{previous_path}: #{e}"))
+
+          unless dry_run?
+            begin
+              FileUtils.rmdir(upgraded_path, parents: true)
+            rescue Errno::ENOTEMPTY
+              # OK
+            end
+          end
+        end
+      end
+
+      # Because we move files style-by-style, it's important to restore
+      # previous version at the end. The upgrade will be recorded after
+      # all styles are updated
+      attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
+    end
+  end
+end
diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb
index ce5780557..f3e51dbd3 100644
--- a/lib/paperclip/attachment_extensions.rb
+++ b/lib/paperclip/attachment_extensions.rb
@@ -14,6 +14,15 @@ module Paperclip
       end
     end
 
+    def storage_schema_version
+      instance_read(:storage_schema_version) || 0
+    end
+
+    def assign_attributes
+      super
+      instance_write(:storage_schema_version, 1)
+    end
+
     def variant?(other_filename)
       return true  if original_filename == other_filename
       return false if original_filename.nil?