From 75e33fd08fb4f926897490b6fa42890b393c0989 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 12 Mar 2022 08:12:57 +0100 Subject: Fix null values being included in some indexes (#17711) * Fix null values being included in some indexes * Update lib/mastodon/migration_helpers.rb Co-authored-by: Claire * Add documentation link to corruption error message Co-authored-by: Claire --- lib/mastodon/migration_helpers.rb | 28 ++++++++++++++++++++++++++-- lib/tasks/db.rake | 15 +-------------- 2 files changed, 27 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 5bc903349..e920ff88f 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -42,8 +42,14 @@ module Mastodon module MigrationHelpers class CorruptionError < StandardError - def initialize(message = nil) - super(message.presence || 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing') + attr_reader :index_name + + def initialize(index_name) + @index_name = index_name + + super "The index `#{index_name}` seems to be corrupted, it contains duplicate rows. " \ + 'For information on how to fix this, see our documentation: ' \ + 'https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/' end def cause @@ -802,6 +808,24 @@ module Mastodon columns(table).find { |column| column.name == name } end + # Update the configuration of an index by creating a new one and then + # removing the old one + def update_index(table_name, index_name, columns, **index_options) + if index_name_exists?(table_name, "#{index_name}_new") && index_name_exists?(table_name, index_name) + remove_index table_name, "#{index_name}_new" + end + + begin + add_index table_name, columns, **index_options.merge(name: "#{index_name}_new", algorithm: :concurrently) + rescue ActiveRecord::RecordNotUnique + remove_index table_name, name: "#{index_name}_new" + raise CorruptionError.new(index_name) + end + + remove_index table_name, name: index_name if index_name_exists?(table_name, index_name) + rename_index table_name, "#{index_name}_new", index_name + end + # This will replace the first occurrence of a string in a column with # the replacement # On postgresql we can use `regexp_replace` for that. diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index a6b8c74cd..ca939fd1f 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -17,23 +17,10 @@ namespace :db do end end - task :post_migration_hook do - at_exit do - unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate']) - warn <<~WARNING - Your database collation may be susceptible to index corruption. - (This warning does not indicate that index corruption has occurred, and it can be ignored if you've previously checked for index corruption) - (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/) - WARNING - end - end - end - task :pre_migration_check do version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i - abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500 + abort 'This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon' if version < 90_500 end Rake::Task['db:migrate'].enhance(['db:pre_migration_check']) - Rake::Task['db:migrate'].enhance(['db:post_migration_hook']) end -- cgit From 642528f45513acb823a7291f32fb07377bd2b621 Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 12 Mar 2022 08:33:11 +0100 Subject: Update fix-duplicates maintenance task (#17731) * Update fix-duplicates task to 2022_02_10_153119 Also add support for Appeal to AccountMerging#merge_with! * Update fix-duplicates task to 2022_03_07_094650 * Update fix-duplicates task to 2022_03_09_213005 * Update fix-duplicates task to 2022_03_07_083603 * Update fix-duplicates task to 2022_03_10_060626 * Update fix-duplicates script to 2022_03_07_083603 * Update fix-duplicates task to 2022_03_10_060706 * Update fix-duplicates task to 2022_03_10_060959 * Silence CodeClimate --- .rubocop.yml | 3 +++ app/models/concerns/account_merging.rb | 7 +++++- lib/mastodon/maintenance_cli.rb | 41 +++++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/.rubocop.yml b/.rubocop.yml index 68634e9e3..4948aea5a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -29,6 +29,9 @@ Layout/EmptyLineAfterMagicComment: Layout/EmptyLineAfterGuardClause: Enabled: false +Layout/EmptyLineBetweenDefs: + AllowAdjacentOneLineDefs: true + Layout/EmptyLinesAroundAttributeAccessor: Enabled: true diff --git a/app/models/concerns/account_merging.rb b/app/models/concerns/account_merging.rb index 119773e6b..8161761fb 100644 --- a/app/models/concerns/account_merging.rb +++ b/app/models/concerns/account_merging.rb @@ -15,7 +15,8 @@ module AccountMerging Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite, Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin, AccountStat, ListAccount, - PollVote, Mention, AccountDeletionRequest, AccountNote, FollowRecommendationSuppression + PollVote, Mention, AccountDeletionRequest, AccountNote, FollowRecommendationSuppression, + Appeal ] owned_classes.each do |klass| @@ -47,6 +48,10 @@ module AccountMerging record.update_attribute(:reference_account_id, id) end + Appeal.where(account_warning_id: other_account.id).find_each do |record| + record.update_attribute(:account_warning_id, id) + end + # Some follow relationships have moved, so the cache is stale Rails.cache.delete_matched("followers_hash:#{id}:*") Rails.cache.delete_matched("relationships:#{id}:*") diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 1e047d96c..33beec236 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -13,8 +13,8 @@ module Mastodon true end - MIN_SUPPORTED_VERSION = 2019_10_01_213028 - MAX_SUPPORTED_VERSION = 2022_01_18_183123 + MIN_SUPPORTED_VERSION = 2019_10_01_213028 # rubocop:disable Style/NumericLiterals + MAX_SUPPORTED_VERSION = 2022_03_10_060959 # rubocop:disable Style/NumericLiterals # Stubs to enjoy ActiveRecord queries while not depending on a particular # version of the code/database @@ -44,6 +44,7 @@ module Mastodon class WebauthnCredential < ApplicationRecord; end class FollowRecommendationSuppression < ApplicationRecord; end class CanonicalEmailBlock < ApplicationRecord; end + class Appeal < ApplicationRecord; end class PreviewCard < ApplicationRecord self.inheritance_column = false @@ -92,6 +93,7 @@ module Mastodon owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions) owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs) + owned_classes << Appeal if ActiveRecord::Base.connection.table_exists?(:appeals) owned_classes.each do |klass| klass.where(account_id: other_account.id).find_each do |record| @@ -121,6 +123,12 @@ module Mastodon record.update_attribute(:reference_account_id, id) end end + + if ActiveRecord::Base.connection.table_exists?(:appeals) + Appeal.where(account_warning_id: other_account.id).find_each do |record| + record.update_attribute(:account_warning_id, id) + end + end end end @@ -199,7 +207,7 @@ module Mastodon end @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…' - if ActiveRecord::Migrator.current_version < 20200620164023 + if ActiveRecord::Migrator.current_version < 20200620164023 # rubocop:disable Style/NumericLiterals ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true else ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true @@ -242,7 +250,7 @@ module Mastodon end end - if ActiveRecord::Migrator.current_version < 20220118183010 + if ActiveRecord::Migrator.current_version < 20220118183010 # rubocop:disable Style/NumericLiterals ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" @@ -266,7 +274,12 @@ module Mastodon ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010 - ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true + + if ActiveRecord::Migrator.current_version < 20220310060641 # rubocop:disable Style/NumericLiterals + ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true + else + ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops + end end def deduplicate_account_domain_blocks! @@ -325,7 +338,11 @@ module Mastodon end @prompt.say 'Restoring conversations indexes…' - ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true + if ActiveRecord::Migrator.current_version < 20220307083603 # rubocop:disable Style/NumericLiterals + ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true + else + ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops + end end def deduplicate_custom_emojis! @@ -438,7 +455,11 @@ module Mastodon end @prompt.say 'Restoring media_attachments indexes…' - ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true + if ActiveRecord::Migrator.current_version < 20220310060626 # rubocop:disable Style/NumericLiterals + ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true + else + ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops + end end def deduplicate_preview_cards! @@ -467,7 +488,11 @@ module Mastodon end @prompt.say 'Restoring statuses indexes…' - ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true + if ActiveRecord::Migrator.current_version < 20220310060706 # rubocop:disable Style/NumericLiterals + ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true + else + ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops + end end def deduplicate_tags! -- cgit