Merge pull request #1716 from ClearlyClaire/glitch-soc/merge-upstream

Merge upstream changes

3 files changed, 60 insertions, 24 deletions

diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb
index 1e047d96c..33beec236 100644
--- a/lib/mastodon/maintenance_cli.rb
+++ b/lib/mastodon/maintenance_cli.rb
@@ -13,8 +13,8 @@ module Mastodon
       true
     end
 
-    MIN_SUPPORTED_VERSION = 2019_10_01_213028
-    MAX_SUPPORTED_VERSION = 2022_01_18_183123
+    MIN_SUPPORTED_VERSION = 2019_10_01_213028 # rubocop:disable Style/NumericLiterals
+    MAX_SUPPORTED_VERSION = 2022_03_10_060959 # rubocop:disable Style/NumericLiterals
 
     # Stubs to enjoy ActiveRecord queries while not depending on a particular
     # version of the code/database
@@ -44,6 +44,7 @@ module Mastodon
     class WebauthnCredential < ApplicationRecord; end
     class FollowRecommendationSuppression < ApplicationRecord; end
     class CanonicalEmailBlock < ApplicationRecord; end
+    class Appeal < ApplicationRecord; end
 
     class PreviewCard < ApplicationRecord
       self.inheritance_column = false
@@ -92,6 +93,7 @@ module Mastodon
         owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
         owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions)
         owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs)
+        owned_classes << Appeal if ActiveRecord::Base.connection.table_exists?(:appeals)
 
         owned_classes.each do |klass|
           klass.where(account_id: other_account.id).find_each do |record|
@@ -121,6 +123,12 @@ module Mastodon
             record.update_attribute(:reference_account_id, id)
           end
         end
+
+        if ActiveRecord::Base.connection.table_exists?(:appeals)
+          Appeal.where(account_warning_id: other_account.id).find_each do |record|
+            record.update_attribute(:account_warning_id, id)
+          end
+        end
       end
     end
 
@@ -199,7 +207,7 @@ module Mastodon
       end
 
       @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
-      if ActiveRecord::Migrator.current_version < 20200620164023
+      if ActiveRecord::Migrator.current_version < 20200620164023 # rubocop:disable Style/NumericLiterals
         ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
       else
         ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
@@ -242,7 +250,7 @@ module Mastodon
         end
       end
 
-      if ActiveRecord::Migrator.current_version < 20220118183010
+      if ActiveRecord::Migrator.current_version < 20220118183010 # rubocop:disable Style/NumericLiterals
         ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
           users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
           @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
@@ -266,7 +274,12 @@ module Mastodon
       ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
       ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
       ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010
-      ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
+
+      if ActiveRecord::Migrator.current_version < 20220310060641 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_account_domain_blocks!
@@ -325,7 +338,11 @@ module Mastodon
       end
 
       @prompt.say 'Restoring conversations indexes…'
-      ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
+      if ActiveRecord::Migrator.current_version < 20220307083603 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_custom_emojis!
@@ -438,7 +455,11 @@ module Mastodon
       end
 
       @prompt.say 'Restoring media_attachments indexes…'
-      ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
+      if ActiveRecord::Migrator.current_version < 20220310060626 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_preview_cards!
@@ -467,7 +488,11 @@ module Mastodon
       end
 
       @prompt.say 'Restoring statuses indexes…'
-      ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
+      if ActiveRecord::Migrator.current_version < 20220310060706 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_tags!
diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb
index 5bc903349..e920ff88f 100644
--- a/lib/mastodon/migration_helpers.rb
+++ b/lib/mastodon/migration_helpers.rb
@@ -42,8 +42,14 @@
 module Mastodon
   module MigrationHelpers
     class CorruptionError < StandardError
-      def initialize(message = nil)
-        super(message.presence || 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing')
+      attr_reader :index_name
+
+      def initialize(index_name)
+        @index_name = index_name
+
+        super "The index `#{index_name}` seems to be corrupted, it contains duplicate rows. " \
+          'For information on how to fix this, see our documentation: ' \
+          'https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/'
       end
 
       def cause
@@ -802,6 +808,24 @@ module Mastodon
       columns(table).find { |column| column.name == name }
     end
 
+    # Update the configuration of an index by creating a new one and then
+    # removing the old one
+    def update_index(table_name, index_name, columns, **index_options)
+      if index_name_exists?(table_name, "#{index_name}_new") && index_name_exists?(table_name, index_name)
+        remove_index table_name, "#{index_name}_new"
+      end
+
+      begin
+        add_index table_name, columns, **index_options.merge(name: "#{index_name}_new", algorithm: :concurrently)
+      rescue ActiveRecord::RecordNotUnique
+        remove_index table_name, name: "#{index_name}_new"
+        raise CorruptionError.new(index_name)
+      end
+
+      remove_index table_name, name: index_name if index_name_exists?(table_name, index_name)
+      rename_index table_name, "#{index_name}_new", index_name
+    end
+
     # This will replace the first occurrence of a string in a column with
     # the replacement
     # On postgresql we can use `regexp_replace` for that.
diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index a6b8c74cd..ca939fd1f 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -17,23 +17,10 @@ namespace :db do
     end
   end
 
-  task :post_migration_hook do
-    at_exit do
-      unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate'])
-        warn <<~WARNING
-          Your database collation may be susceptible to index corruption.
-            (This warning does not indicate that index corruption has occurred, and it can be ignored if you've previously checked for index corruption)
-            (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/)
-        WARNING
-      end
-    end
-  end
-
   task :pre_migration_check do
     version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
-    abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500
+    abort 'This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon' if version < 90_500
   end
 
   Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])
-  Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
 end