From b715bd8e53537c44915fa76e89e4af53486b027f Mon Sep 17 00:00:00 2001 From: 9p4 Date: Wed, 8 Mar 2023 11:06:53 -0500 Subject: Add refreshing many accounts at once with "tootctl accounts refresh" (#23304) --- lib/mastodon/accounts_cli.rb | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 98855cbd0..a6532541e 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -372,16 +372,16 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean - desc 'refresh [USERNAME]', 'Fetch remote user data and files' + desc 'refresh [USERNAMES]', 'Fetch remote user data and files' long_desc <<-LONG_DESC Fetch remote user data and files for one or multiple accounts. With the --all option, all remote accounts will be processed. Through the --domain option, this can be narrowed down to a - specific domain only. Otherwise, a single remote account must - be specified with USERNAME. + specific domain only. Otherwise, remote accounts must be + specified with space-separated USERNAMES. LONG_DESC - def refresh(username = nil) + def refresh(*usernames) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' if options[:domain] || options[:all] @@ -397,19 +397,25 @@ module Mastodon end say("Refreshed #{processed} accounts#{dry_run}", :green, true) - elsif username.present? - username, domain = username.split('@') - account = Account.find_remote(username, domain) + elsif !usernames.empty? + usernames.each do |user| + user, domain = user.split('@') + account = Account.find_remote(user, domain) + + if account.nil? + say('No such account', :red) + exit(1) + end - if account.nil? - say('No such account', :red) - exit(1) - end + next if options[:dry_run] - unless options[:dry_run] - account.reset_avatar! - account.reset_header! - account.save + begin + account.reset_avatar! + account.reset_header! + account.save + rescue Mastodon::UnexpectedResponseError + say("Account failed: #{user}@#{domain}", :red) + end end say("OK#{dry_run}", :green) -- cgit From f432db7b9f54501269c6377faf85b1b4b1424239 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 12 Mar 2023 23:47:55 +0100 Subject: Fix sidekiq jobs not triggering Elasticsearch index updates (#24046) --- bin/tootctl | 4 +++- config/initializers/chewy.rb | 1 - lib/mastodon/sidekiq_middleware.rb | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/bin/tootctl b/bin/tootctl index a9ebb22c6..9c7ae8b87 100755 --- a/bin/tootctl +++ b/bin/tootctl @@ -5,7 +5,9 @@ require_relative '../config/boot' require_relative '../lib/cli' begin - Mastodon::CLI.start(ARGV) + Chewy.strategy(:mastodon) do + Mastodon::CLI.start(ARGV) + end rescue Interrupt exit(130) end diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index 752fc3c6d..2a91efefc 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -19,7 +19,6 @@ Chewy.settings = { # cycle, which takes care of checking if Elasticsearch is enabled # or not. However, mind that for the Rails console, the :urgent # strategy is set automatically with no way to override it. -Chewy.root_strategy = :mastodon Chewy.request_strategy = :mastodon Chewy.use_after_commit_callbacks = false diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index c75e8401f..9832e1a27 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -3,8 +3,8 @@ class Mastodon::SidekiqMiddleware BACKTRACE_LIMIT = 3 - def call(*) - yield + def call(*, &block) + Chewy.strategy(:mastodon, &block) rescue Mastodon::HostValidationError # Do not retry rescue => e -- cgit From e762a14c0ab235c337b3cddbef1eaf72b26faf58 Mon Sep 17 00:00:00 2001 From: Nick Schonning Date: Sun, 12 Mar 2023 19:03:07 -0400 Subject: Enable Rubocop Performance/DeleteSuffix (#24077) --- .rubocop_todo.yml | 7 ------- lib/tasks/repo.rake | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index a9727f939..8a96e59f9 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -301,13 +301,6 @@ Performance/DeletePrefix: - 'app/services/resolve_account_service.rb' - 'app/services/tag_search_service.rb' -# Offense count: 1 -# This cop supports unsafe autocorrection (--autocorrect-all). -# Configuration parameters: SafeMultiline. -Performance/DeleteSuffix: - Exclude: - - 'lib/tasks/repo.rake' - # Offense count: 19 # This cop supports unsafe autocorrection (--autocorrect-all). Performance/MapCompact: diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake index 5d1b4f754..888337b4f 100644 --- a/lib/tasks/repo.rake +++ b/lib/tasks/repo.rake @@ -91,8 +91,8 @@ namespace :repo do missing_json_files = I18n.available_locales.reject { |locale| Rails.root.join('app', 'javascript', 'mastodon', 'locales', "#{locale}.json").exist? } locales_in_files = Dir[Rails.root.join('config', 'locales', '*.yml')].map do |path| - file_name = File.basename(path) - file_name.gsub(/\A(doorkeeper|devise|activerecord|simple_form)\./, '').gsub(/\.yml\z/, '').to_sym + file_name = File.basename(path, '.yml') + file_name.gsub(/\A(doorkeeper|devise|activerecord|simple_form)\./, '').to_sym end.uniq.compact missing_available_locales = locales_in_files - I18n.available_locales -- cgit From 1d0ad558ff51ce3e42ef4e4c9a9bed3c4b722d0d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 Mar 2023 03:45:15 +0100 Subject: Change sidekiq-bulk's batch size from 10,000 to 1,000 jobs in one Redis call (#24034) --- app/lib/activitypub/forwarder.rb | 2 +- app/services/delete_account_service.rb | 4 ++-- app/services/remove_status_service.rb | 2 +- app/services/suspend_account_service.rb | 12 ++++++------ app/services/unsuspend_account_service.rb | 2 +- app/services/update_account_service.rb | 2 +- app/workers/activitypub/distribute_poll_update_worker.rb | 2 +- app/workers/activitypub/move_distribution_worker.rb | 2 +- app/workers/activitypub/raw_distribution_worker.rb | 2 +- lib/cli.rb | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/app/lib/activitypub/forwarder.rb b/app/lib/activitypub/forwarder.rb index b01d63e58..3a94f9669 100644 --- a/app/lib/activitypub/forwarder.rb +++ b/app/lib/activitypub/forwarder.rb @@ -12,7 +12,7 @@ class ActivityPub::Forwarder end def forward! - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, signature_account_id, inbox_url] end end diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index a2d535d26..190a72e5c 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -257,11 +257,11 @@ class DeleteAccountService < BaseService end def delete_actor! - ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes, limit: 1_000) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end - ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes) do |inbox_url| + ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes, limit: 1_000) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 45cfb75f4..ea799db57 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -88,7 +88,7 @@ class RemoveStatusService < BaseService status_reach_finder = StatusReachFinder.new(@status, unsafe: true) - ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 211544fea..cfb3eb583 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -31,13 +31,13 @@ class SuspendAccountService < BaseService # counterpart to this operation, i.e. you can't then force a remote # account to re-follow you, so this part is not reversible. - follows = Follow.where(account: @account).to_a + Follow.where(account: @account).find_in_batches do |follows| + ActivityPub::DeliveryWorker.push_bulk(follows) do |follow| + [Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), follow.target_account_id, @account.inbox_url] + end - ActivityPub::DeliveryWorker.push_bulk(follows) do |follow| - [Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), follow.target_account_id, @account.inbox_url] + follows.each(&:destroy) end - - follows.each(&:destroy) end def distribute_update_actor! @@ -45,7 +45,7 @@ class SuspendAccountService < BaseService account_reach_finder = AccountReachFinder.new(@account) - ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb index 70667308e..d851a0f70 100644 --- a/app/services/unsuspend_account_service.rb +++ b/app/services/unsuspend_account_service.rb @@ -41,7 +41,7 @@ class UnsuspendAccountService < BaseService account_reach_finder = AccountReachFinder.new(@account) - ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/update_account_service.rb b/app/services/update_account_service.rb index 71976ab00..4604d71b2 100644 --- a/app/services/update_account_service.rb +++ b/app/services/update_account_service.rb @@ -22,7 +22,7 @@ class UpdateAccountService < BaseService def authorize_all_follow_requests(account) follow_requests = FollowRequest.where(target_account: account) follow_requests = follow_requests.preload(:account).select { |req| !req.account.silenced? } - AuthorizeFollowWorker.push_bulk(follow_requests) do |req| + AuthorizeFollowWorker.push_bulk(follow_requests, limit: 1_000) do |req| [req.account_id, req.target_account_id] end end diff --git a/app/workers/activitypub/distribute_poll_update_worker.rb b/app/workers/activitypub/distribute_poll_update_worker.rb index 25dee4ee2..8c1eefd93 100644 --- a/app/workers/activitypub/distribute_poll_update_worker.rb +++ b/app/workers/activitypub/distribute_poll_update_worker.rb @@ -12,7 +12,7 @@ class ActivityPub::DistributePollUpdateWorker return unless @status.preloadable_poll - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, @account.id, inbox_url] end diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb index 65c5c0d1c..1680fcc76 100644 --- a/app/workers/activitypub/move_distribution_worker.rb +++ b/app/workers/activitypub/move_distribution_worker.rb @@ -10,7 +10,7 @@ class ActivityPub::MoveDistributionWorker @migration = AccountMigration.find(migration_id) @account = @migration.account - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [signed_payload, @account.id, inbox_url] end diff --git a/app/workers/activitypub/raw_distribution_worker.rb b/app/workers/activitypub/raw_distribution_worker.rb index 8ecc17db9..c77821e0f 100644 --- a/app/workers/activitypub/raw_distribution_worker.rb +++ b/app/workers/activitypub/raw_distribution_worker.rb @@ -25,7 +25,7 @@ class ActivityPub::RawDistributionWorker def distribute! return if inboxes.empty? - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, source_account_id, inbox_url, options] end end diff --git a/lib/cli.rb b/lib/cli.rb index 157465c4b..ac235cf03 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -131,7 +131,7 @@ module Mastodon json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account)) unless options[:dry_run] - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [json, account.id, inbox_url] end -- cgit