diff options
Diffstat (limited to 'lib/mastodon')
| -rw-r--r-- | lib/mastodon/accounts_cli.rb | 66 | ||||
| -rw-r--r-- | lib/mastodon/cli_helper.rb | 24 | ||||
| -rw-r--r-- | lib/mastodon/domains_cli.rb | 83 | ||||
| -rw-r--r-- | lib/mastodon/emoji_cli.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/feeds_cli.rb | 6 | ||||
| -rw-r--r-- | lib/mastodon/ip_blocks_cli.rb | 12 | ||||
| -rw-r--r-- | lib/mastodon/maintenance_cli.rb | 94 | ||||
| -rw-r--r-- | lib/mastodon/media_cli.rb | 18 | ||||
| -rw-r--r-- | lib/mastodon/migration_helpers.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/migration_warning.rb | 55 | ||||
| -rw-r--r-- | lib/mastodon/premailer_webpack_strategy.rb | 2 | ||||
| -rw-r--r-- | lib/mastodon/redis_config.rb | 22 | ||||
| -rw-r--r-- | lib/mastodon/search_cli.rb | 12 | ||||
| -rw-r--r-- | lib/mastodon/sidekiq_middleware.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/snowflake.rb | 2 | ||||
| -rw-r--r-- | lib/mastodon/statuses_cli.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/upgrade_cli.rb | 18 | ||||
| -rw-r--r-- | lib/mastodon/version.rb | 2 |
18 files changed, 246 insertions, 186 deletions
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 34afbc699..a6532541e 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -372,16 +372,16 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean - desc 'refresh [USERNAME]', 'Fetch remote user data and files' + desc 'refresh [USERNAMES]', 'Fetch remote user data and files' long_desc <<-LONG_DESC Fetch remote user data and files for one or multiple accounts. With the --all option, all remote accounts will be processed. Through the --domain option, this can be narrowed down to a - specific domain only. Otherwise, a single remote account must - be specified with USERNAME. + specific domain only. Otherwise, remote accounts must be + specified with space-separated USERNAMES. LONG_DESC - def refresh(username = nil) + def refresh(*usernames) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' if options[:domain] || options[:all] @@ -397,19 +397,25 @@ module Mastodon end say("Refreshed #{processed} accounts#{dry_run}", :green, true) - elsif username.present? - username, domain = username.split('@') - account = Account.find_remote(username, domain) + elsif !usernames.empty? + usernames.each do |user| + user, domain = user.split('@') + account = Account.find_remote(user, domain) + + if account.nil? + say('No such account', :red) + exit(1) + end - if account.nil? - say('No such account', :red) - exit(1) - end + next if options[:dry_run] - unless options[:dry_run] - account.reset_avatar! - account.reset_header! - account.save + begin + account.reset_avatar! + account.reset_header! + account.save + rescue Mastodon::UnexpectedResponseError + say("Account failed: #{user}@#{domain}", :red) + end end say("OK#{dry_run}", :green) @@ -490,14 +496,12 @@ module Mastodon scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id)) scope.find_each do |target_account| - begin - UnfollowService.new.call(account, target_account) - rescue => e - progress.log pastel.red("Error processing #{target_account.id}: #{e}") - ensure - progress.increment - processed += 1 - end + UnfollowService.new.call(account, target_account) + rescue => e + progress.log pastel.red("Error processing #{target_account.id}: #{e}") + ensure + progress.increment + processed += 1 end BootstrapTimelineWorker.perform_async(account.id) @@ -507,14 +511,12 @@ module Mastodon scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id)) scope.find_each do |target_account| - begin - UnfollowService.new.call(target_account, account) - rescue => e - progress.log pastel.red("Error processing #{target_account.id}: #{e}") - ensure - progress.increment - processed += 1 - end + UnfollowService.new.call(target_account, account) + rescue => e + progress.log pastel.red("Error processing #{target_account.id}: #{e}") + ensure + progress.increment + processed += 1 end end @@ -631,7 +633,7 @@ module Mastodon exit(1) end - unless options[:force] || migration.target_acount_id == account.moved_to_account_id + unless options[:force] || migration.target_account_id == account.moved_to_account_id say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red) exit(1) end diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index a78a28e27..ab1351ae8 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -42,17 +42,17 @@ module Mastodon items.each do |item| futures << Concurrent::Future.execute(executor: pool) do - begin - if !progress.total.nil? && progress.progress + 1 > progress.total - # The number of items has changed between start and now, - # since there is no good way to predict the final count from - # here, just change the progress bar to an indeterminate one + if !progress.total.nil? && progress.progress + 1 > progress.total + # The number of items has changed between start and now, + # since there is no good way to predict the final count from + # here, just change the progress bar to an indeterminate one - progress.total = nil - end + progress.total = nil + end - progress.log("Processing #{item.id}") if options[:verbose] + progress.log("Processing #{item.id}") if options[:verbose] + Chewy.strategy(:mastodon) do result = ActiveRecord::Base.connection_pool.with_connection do yield(item) ensure @@ -61,11 +61,11 @@ module Mastodon end aggregate.increment(result) if result.is_a?(Integer) - rescue => e - progress.log pastel.red("Error processing #{item.id}: #{e}") - ensure - progress.increment end + rescue => e + progress.log pastel.red("Error processing #{item.id}: #{e}") + ensure + progress.increment end end diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 77364ffbb..05f08f462 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -18,6 +18,8 @@ module Mastodon option :dry_run, type: :boolean option :limited_federation_mode, type: :boolean option :by_uri, type: :boolean + option :include_subdomains, type: :boolean + option :purge_domain_blocks, type: :boolean desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any @@ -33,40 +35,75 @@ module Mastodon that has the handle `foo@bar.com` but whose profile is at the URL `https://mastodon-bar.com/users/foo`, would be purged by either `tootctl domains purge bar.com` or `tootctl domains purge --by-uri mastodon-bar.com`. + + When the --include-subdomains option is given, not only DOMAIN is deleted, but all + subdomains as well. Note that this may be considerably slower. + + When the --purge-domain-blocks option is given, also purge matching domain blocks. LONG_DESC def purge(*domains) - dry_run = options[:dry_run] ? ' (DRY RUN)' : '' - - scope = begin - if options[:limited_federation_mode] - Account.remote.where.not(domain: DomainAllow.pluck(:domain)) - elsif !domains.empty? - if options[:by_uri] - domains.map { |domain| Account.remote.where(Account.arel_table[:uri].matches("https://#{domain}/%", false, true)) }.reduce(:or) - else - Account.remote.where(domain: domains) - end + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) } + account_scope = Account.none + domain_block_scope = DomainBlock.none + emoji_scope = CustomEmoji.none + + # Sanity check on command arguments + if options[:limited_federation_mode] && !domains.empty? + say('DOMAIN parameter not supported with --limited-federation-mode', :red) + exit(1) + elsif domains.empty? && !options[:limited_federation_mode] + say('No domain(s) given', :red) + exit(1) + end + + # Build scopes from command arguments + if options[:limited_federation_mode] + account_scope = Account.remote.where.not(domain: DomainAllow.select(:domain)) + emoji_scope = CustomEmoji.remote.where.not(domain: DomainAllow.select(:domain)) + else + # Handle wildcard subdomains + subdomain_patterns = domains.filter_map { |domain| "%.#{Account.sanitize_sql_like(domain[2..])}" if domain.start_with?('*.') } + domains = domains.filter { |domain| !domain.start_with?('*.') } + # Handle --include-subdomains + subdomain_patterns += domains.map { |domain| "%.#{Account.sanitize_sql_like(domain)}" } if options[:include_subdomains] + uri_patterns = (domains.map { |domain| Account.sanitize_sql_like(domain) } + subdomain_patterns).map { |pattern| "https://#{pattern}/%" } + + if options[:purge_domain_blocks] + domain_block_scope = DomainBlock.where(domain: domains) + domain_block_scope = domain_block_scope.or(DomainBlock.where(DomainBlock.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty? + end + + if options[:by_uri] + account_scope = Account.remote.where(Account.arel_table[:uri].matches_any(uri_patterns, false, true)) + emoji_scope = CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(uri_patterns, false, true)) else - say('No domain(s) given', :red) - exit(1) + account_scope = Account.remote.where(domain: domains) + account_scope = account_scope.or(Account.remote.where(Account.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty? + emoji_scope = CustomEmoji.where(domain: domains) + emoji_scope = emoji_scope.or(CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(subdomain_patterns))) unless subdomain_patterns.empty? end end - processed, = parallelize_with_progress(scope) do |account| + # Actually perform the deletions + processed, = parallelize_with_progress(account_scope) do |account| DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run] end - DomainBlock.where(domain: domains).destroy_all unless options[:dry_run] - say("Removed #{processed} accounts#{dry_run}", :green) - custom_emojis = CustomEmoji.where(domain: domains) - custom_emojis_count = custom_emojis.count - custom_emojis.destroy_all unless options[:dry_run] + if options[:purge_domain_blocks] + domain_block_count = domain_block_scope.count + domain_block_scope.in_batches.destroy_all unless options[:dry_run] + say("Removed #{domain_block_count} domain blocks#{dry_run}", :green) + end + + custom_emojis_count = emoji_scope.count + emoji_scope.in_batches.destroy_all unless options[:dry_run] Instance.refresh unless options[:dry_run] - say("Removed #{custom_emojis_count} custom emojis", :green) + say("Removed #{custom_emojis_count} custom emojis#{dry_run}", :green) end option :concurrency, type: :numeric, default: 50, aliases: [:c] @@ -102,7 +139,7 @@ module Mastodon pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0) - work_unit = ->(domain) do + work_unit = lambda do |domain| next if stats.key?(domain) next if options[:exclude_suspended] && domain.match?(blocked_domains) @@ -111,6 +148,7 @@ module Mastodon begin Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res| next unless res.code == 200 + stats[domain] = Oj.load(res.to_s) end @@ -124,9 +162,10 @@ module Mastodon Request.new(:get, "https://#{domain}/api/v1/instance/activity").perform do |res| next unless res.code == 200 + stats[domain]['activity'] = Oj.load(res.to_s) end - rescue StandardError + rescue failed.increment ensure processed.increment diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index a3e947909..88065c2a3 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -49,7 +49,7 @@ module Mastodon next if filename.start_with?('._') shortcode = [options[:prefix], filename, options[:suffix]].compact.join - custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase) + custom_emoji = CustomEmoji.local.find_by('LOWER(shortcode) = ?', shortcode.downcase) if custom_emoji && !options[:overwrite] skipped += 1 @@ -68,7 +68,7 @@ module Mastodon failed += 1 say('Failure/Error: ', :red) say(entry.full_name) - say(' ' + custom_emoji.errors[:image].join(', '), :red) + say(" #{custom_emoji.errors[:image].join(', ')}", :red) end end end diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb index 428d63a44..fcfb48740 100644 --- a/lib/mastodon/feeds_cli.rb +++ b/lib/mastodon/feeds_cli.rb @@ -53,11 +53,7 @@ module Mastodon desc 'clear', 'Remove all home and list feeds from Redis' def clear keys = redis.keys('feed:*') - - redis.pipelined do - keys.each { |key| redis.del(key) } - end - + redis.del(keys) say('OK', :green) end end diff --git a/lib/mastodon/ip_blocks_cli.rb b/lib/mastodon/ip_blocks_cli.rb index 5c38c1aca..08939c092 100644 --- a/lib/mastodon/ip_blocks_cli.rb +++ b/lib/mastodon/ip_blocks_cli.rb @@ -79,13 +79,11 @@ module Mastodon skipped = 0 addresses.each do |address| - ip_blocks = begin - if options[:force] - IpBlock.where('ip >>= ?', address) - else - IpBlock.where('ip <<= ?', address) - end - end + ip_blocks = if options[:force] + IpBlock.where('ip >>= ?', address) + else + IpBlock.where('ip <<= ?', address) + end if ip_blocks.empty? say("#{address} is not yet blocked", :yellow) diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 85937da81..ff8f6ddda 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -13,8 +13,8 @@ module Mastodon true end - MIN_SUPPORTED_VERSION = 2019_10_01_213028 # rubocop:disable Style/NumericLiterals - MAX_SUPPORTED_VERSION = 2022_11_04_133904 # rubocop:disable Style/NumericLiterals + MIN_SUPPORTED_VERSION = 2019_10_01_213028 + MAX_SUPPORTED_VERSION = 2022_11_04_133904 # Stubs to enjoy ActiveRecord queries while not depending on a particular # version of the code/database @@ -98,11 +98,9 @@ module Mastodon owned_classes.each do |klass| klass.where(account_id: other_account.id).find_each do |record| - begin - record.update_attribute(:account_id, id) - rescue ActiveRecord::RecordNotUnique - next - end + record.update_attribute(:account_id, id) + rescue ActiveRecord::RecordNotUnique + next end end @@ -111,11 +109,9 @@ module Mastodon target_classes.each do |klass| klass.where(target_account_id: other_account.id).find_each do |record| - begin - record.update_attribute(:target_account_id, id) - rescue ActiveRecord::RecordNotUnique - next - end + record.update_attribute(:target_account_id, id) + rescue ActiveRecord::RecordNotUnique + next end end @@ -209,7 +205,7 @@ module Mastodon end @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…' - if ActiveRecord::Migrator.current_version < 20200620164023 # rubocop:disable Style/NumericLiterals + if ActiveRecord::Migrator.current_version < 2020_06_20_164023 ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true else ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true @@ -252,7 +248,7 @@ module Mastodon end end - if ActiveRecord::Migrator.current_version < 20220118183010 # rubocop:disable Style/NumericLiterals + if ActiveRecord::Migrator.current_version < 2022_01_18_183010 ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" @@ -275,9 +271,9 @@ module Mastodon @prompt.say 'Restoring users indexes…' ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true - ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010 + ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 2022_01_18_183010 - if ActiveRecord::Migrator.current_version < 20220310060641 # rubocop:disable Style/NumericLiterals + if ActiveRecord::Migrator.current_version < 2022_03_10_060641 ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true else ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops @@ -293,7 +289,7 @@ module Mastodon end @prompt.say 'Restoring account domain blocks indexes…' - ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true + ActiveRecord::Base.connection.add_index :account_domain_blocks, %w(account_id domain), name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true end def deduplicate_account_identity_proofs! @@ -307,7 +303,7 @@ module Mastodon end @prompt.say 'Restoring account identity proofs indexes…' - ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true + ActiveRecord::Base.connection.add_index :account_identity_proofs, %w(account_id provider provider_username), name: 'index_account_proofs_on_account_and_provider_and_username', unique: true end def deduplicate_announcement_reactions! @@ -321,7 +317,7 @@ module Mastodon end @prompt.say 'Restoring announcement_reactions indexes…' - ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true + ActiveRecord::Base.connection.add_index :announcement_reactions, %w(account_id announcement_id name), name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true end def deduplicate_conversations! @@ -340,7 +336,7 @@ module Mastodon end @prompt.say 'Restoring conversations indexes…' - if ActiveRecord::Migrator.current_version < 20220307083603 # rubocop:disable Style/NumericLiterals + if ActiveRecord::Migrator.current_version < 2022_03_07_083603 ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true else ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops @@ -363,7 +359,7 @@ module Mastodon end @prompt.say 'Restoring custom_emojis indexes…' - ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true + ActiveRecord::Base.connection.add_index :custom_emojis, %w(shortcode domain), name: 'index_custom_emojis_on_shortcode_and_domain', unique: true end def deduplicate_custom_emoji_categories! @@ -457,7 +453,7 @@ module Mastodon end @prompt.say 'Restoring media_attachments indexes…' - if ActiveRecord::Migrator.current_version < 20220310060626 # rubocop:disable Style/NumericLiterals + if ActiveRecord::Migrator.current_version < 2022_03_10_060626 ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true else ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops @@ -490,7 +486,7 @@ module Mastodon end @prompt.say 'Restoring statuses indexes…' - if ActiveRecord::Migrator.current_version < 20220310060706 # rubocop:disable Style/NumericLiterals + if ActiveRecord::Migrator.current_version < 2022_03_10_060706 ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true else ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops @@ -512,7 +508,7 @@ module Mastodon end @prompt.say 'Restoring tags indexes…' - if ActiveRecord::Migrator.current_version < 20210421121431 + if ActiveRecord::Migrator.current_version < 2021_04_21_121431 ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true else ActiveRecord::Base.connection.execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)' @@ -554,7 +550,7 @@ module Mastodon @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.' accounts.each_with_index do |account, idx| - @prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A'] + @prompt.say format('%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s', idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A') end @prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.' @@ -601,11 +597,9 @@ module Mastodon owned_classes = [ConversationMute, AccountConversation] owned_classes.each do |klass| klass.where(conversation_id: duplicate_conv.id).find_each do |record| - begin - record.update_attribute(:account_id, main_conv.id) - rescue ActiveRecord::RecordNotUnique - next - end + record.update_attribute(:account_id, main_conv.id) + rescue ActiveRecord::RecordNotUnique + next end end end @@ -629,47 +623,37 @@ module Mastodon owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks) owned_classes.each do |klass| klass.where(status_id: duplicate_status.id).find_each do |record| - begin - record.update_attribute(:status_id, main_status.id) - rescue ActiveRecord::RecordNotUnique - next - end - end - end - - StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record| - begin record.update_attribute(:status_id, main_status.id) rescue ActiveRecord::RecordNotUnique next end end + StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record| + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + Status.where(in_reply_to_id: duplicate_status.id).find_each do |record| - begin - record.update_attribute(:in_reply_to_id, main_status.id) - rescue ActiveRecord::RecordNotUnique - next - end + record.update_attribute(:in_reply_to_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next end Status.where(reblog_of_id: duplicate_status.id).find_each do |record| - begin - record.update_attribute(:reblog_of_id, main_status.id) - rescue ActiveRecord::RecordNotUnique - next - end + record.update_attribute(:reblog_of_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next end end def merge_tags!(main_tag, duplicate_tag) [FeaturedTag].each do |klass| klass.where(tag_id: duplicate_tag.id).find_each do |record| - begin - record.update_attribute(:tag_id, main_tag.id) - rescue ActiveRecord::RecordNotUnique - next - end + record.update_attribute(:tag_id, main_tag.id) + rescue ActiveRecord::RecordNotUnique + next end end end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 24cc98964..b2dfe58d5 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -35,7 +35,6 @@ module Mastodon follow status. By default, only accounts that are not followed by or following anyone locally are pruned. DESC - # rubocop:disable Metrics/PerceivedComplexity def remove if options[:prune_profiles] && options[:remove_headers] say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true) @@ -116,13 +115,11 @@ module Mastodon loop do objects = begin - begin - bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x } - rescue => e - progress.log(pastel.red("Error fetching list of files: #{e}")) - progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key - break - end + bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x } + rescue => e + progress.log(pastel.red("Error fetching list of files: #{e}")) + progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key + break end break if objects.empty? @@ -226,7 +223,6 @@ module Mastodon say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true) end - # rubocop:enable Metrics/PerceivedComplexity option :account, type: :string option :domain, type: :string @@ -277,9 +273,7 @@ module Mastodon exit(1) end - if options[:days].present? - scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) - end + scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) if options[:days].present? processed, aggregate = parallelize_with_progress(scope) do |media_attachment| next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 2ab8150ec..5a252b351 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -289,8 +289,6 @@ module Mastodon # determines this method to be too complex while there's no way to make it # less "complex" without introducing extra methods (which actually will # make things _more_ complex). - # - # rubocop: disable Metrics/AbcSize def update_column_in_batches(table_name, column, value) if transaction_open? raise 'update_column_in_batches can not be run inside a transaction, ' \ @@ -573,7 +571,7 @@ module Mastodon o.conname as name, o.confdeltype as on_delete from pg_constraint o - left join pg_class f on f.oid = o.confrelid + left join pg_class f on f.oid = o.confrelid left join pg_class c on c.oid = o.conrelid left join pg_class m on m.oid = o.conrelid where o.contype = 'f' diff --git a/lib/mastodon/migration_warning.rb b/lib/mastodon/migration_warning.rb new file mode 100644 index 000000000..227f6705d --- /dev/null +++ b/lib/mastodon/migration_warning.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Mastodon + module MigrationWarning + WARNING_SECONDS = 10 + + DEFAULT_WARNING = <<~WARNING_MESSAGE + WARNING: This migration may take a *long* time for large instances. + It will *not* lock tables for any significant time, but it may run + for a very long time. We will pause for #{WARNING_SECONDS} seconds to allow you to + interrupt this migration if you are not ready. + WARNING_MESSAGE + + def migration_duration_warning(explanation = nil) + return unless valid_environment? + + announce_warning(explanation) + + announce_countdown + end + + private + + def announce_countdown + WARNING_SECONDS.downto(1) do |i| + say "Continuing in #{i} second#{i == 1 ? '' : 's'}...", true + sleep 1 + end + end + + def valid_environment? + $stdout.isatty && Rails.env.production? + end + + def announce_warning(explanation) + announce_message prepare_message(explanation) + end + + def announce_message(text) + say '' + text.each_line do |line| + say(line) + end + say '' + end + + def prepare_message(explanation) + if explanation.blank? + DEFAULT_WARNING + else + DEFAULT_WARNING + "\n#{explanation}" + end + end + end +end diff --git a/lib/mastodon/premailer_webpack_strategy.rb b/lib/mastodon/premailer_webpack_strategy.rb index 56ef09c1a..5c297d4d0 100644 --- a/lib/mastodon/premailer_webpack_strategy.rb +++ b/lib/mastodon/premailer_webpack_strategy.rb @@ -13,7 +13,7 @@ module PremailerWebpackStrategy HTTP.get(url).to_s else url = url[1..-1] if url.start_with?('/') - File.read(Rails.root.join('public', url)) + Rails.public_path.join(url).read end css.gsub(/url\(\//, "url(#{asset_host}/") diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index 3522fa11e..2dbe76f83 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -1,17 +1,17 @@ # frozen_string_literal: true def setup_redis_env_url(prefix = nil, defaults = true) - prefix = prefix.to_s.upcase + '_' unless prefix.nil? + prefix = "#{prefix.to_s.upcase}_" unless prefix.nil? prefix = '' if prefix.nil? - return if ENV[prefix + 'REDIS_URL'].present? + return if ENV["#{prefix}REDIS_URL"].present? - password = ENV.fetch(prefix + 'REDIS_PASSWORD') { '' if defaults } - host = ENV.fetch(prefix + 'REDIS_HOST') { 'localhost' if defaults } - port = ENV.fetch(prefix + 'REDIS_PORT') { 6379 if defaults } - db = ENV.fetch(prefix + 'REDIS_DB') { 0 if defaults } + password = ENV.fetch("#{prefix}REDIS_PASSWORD") { '' if defaults } + host = ENV.fetch("#{prefix}REDIS_HOST") { 'localhost' if defaults } + port = ENV.fetch("#{prefix}REDIS_PORT") { 6379 if defaults } + db = ENV.fetch("#{prefix}REDIS_DB") { 0 if defaults } - ENV[prefix + 'REDIS_URL'] = begin + ENV["#{prefix}REDIS_URL"] = begin if [password, host, port, db].all?(&:nil?) ENV['REDIS_URL'] else @@ -27,7 +27,7 @@ setup_redis_env_url(:cache, false) setup_redis_env_url(:sidekiq, false) namespace = ENV.fetch('REDIS_NAMESPACE', nil) -cache_namespace = namespace ? namespace + '_cache' : 'cache' +cache_namespace = namespace ? "#{namespace}_cache" : 'cache' sidekiq_namespace = namespace REDIS_CACHE_PARAMS = { @@ -35,7 +35,7 @@ REDIS_CACHE_PARAMS = { url: ENV['CACHE_REDIS_URL'], expires_in: 10.minutes, namespace: cache_namespace, - pool_size: Sidekiq.server? ? Sidekiq.options[:concurrency] : Integer(ENV['MAX_THREADS'] || 5), + pool_size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5), pool_timeout: 5, connect_timeout: 5, }.freeze @@ -46,6 +46,4 @@ REDIS_SIDEKIQ_PARAMS = { namespace: sidekiq_namespace, }.freeze -if Rails.env.test? - ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" -end +ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test? diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index b206854ab..31e9a3d5a 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -43,13 +43,11 @@ module Mastodon exit(1) end - indices = begin - if options[:only] - options[:only].map { |str| "#{str.camelize}Index".constantize } - else - INDICES - end - end + indices = if options[:only] + options[:only].map { |str| "#{str.camelize}Index".constantize } + else + INDICES + end pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10) importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) } diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index c75e8401f..9832e1a27 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -3,8 +3,8 @@ class Mastodon::SidekiqMiddleware BACKTRACE_LIMIT = 3 - def call(*) - yield + def call(*, &block) + Chewy.strategy(:mastodon, &block) rescue Mastodon::HostValidationError # Do not retry rescue => e diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb index fe0dc1722..8030288af 100644 --- a/lib/mastodon/snowflake.rb +++ b/lib/mastodon/snowflake.rb @@ -115,7 +115,7 @@ module Mastodon::Snowflake # And only those that are using timestamp_id. next unless (data = DEFAULT_REGEX.match(id_col.default_function)) - seq_name = data[:seq_prefix] + '_id_seq' + seq_name = "#{data[:seq_prefix]}_id_seq" # If we were on Postgres 9.5+, we could do CREATE SEQUENCE IF # NOT EXISTS, but we can't depend on that. Instead, catch the diff --git a/lib/mastodon/statuses_cli.rb b/lib/mastodon/statuses_cli.rb index d4c2e6cf2..baab83e29 100644 --- a/lib/mastodon/statuses_cli.rb +++ b/lib/mastodon/statuses_cli.rb @@ -93,7 +93,7 @@ module Mastodon c.table_name = 'statuses_to_be_deleted' end - Object.const_set('StatusToBeDeleted', klass) + Object.const_set(:StatusToBeDeleted, klass) scope = StatusToBeDeleted processed = 0 @@ -175,7 +175,7 @@ module Mastodon c.table_name = 'conversations_to_be_deleted' end - Object.const_set('ConversationsToBeDeleted', klass) + Object.const_set(:ConversationsToBeDeleted, klass) scope = ConversationsToBeDeleted processed = 0 diff --git a/lib/mastodon/upgrade_cli.rb b/lib/mastodon/upgrade_cli.rb index 570b7e6fa..2b60f9eee 100644 --- a/lib/mastodon/upgrade_cli.rb +++ b/lib/mastodon/upgrade_cli.rb @@ -50,16 +50,14 @@ module Mastodon styles << :original unless styles.include?(:original) styles.each do |style| - success = begin - case Paperclip::Attachment.default_options[:storage] - when :s3 - upgrade_storage_s3(progress, attachment, style) - when :fog - upgrade_storage_fog(progress, attachment, style) - when :filesystem - upgrade_storage_filesystem(progress, attachment, style) - end - end + success = case Paperclip::Attachment.default_options[:storage] + when :s3 + upgrade_storage_s3(progress, attachment, style) + when :fog + upgrade_storage_fog(progress, attachment, style) + when :filesystem + upgrade_storage_filesystem(progress, attachment, style) + end upgraded = true if style == :original && success diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 3340d0bf5..603e2b88d 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,7 +9,7 @@ module Mastodon end def minor - 0 + 1 end def patch |