diff options
Diffstat (limited to 'app/services')
| -rw-r--r-- | app/services/activitypub/fetch_remote_account_service.rb | 2 | ||||
| -rw-r--r-- | app/services/activitypub/process_account_service.rb | 28 | ||||
| -rw-r--r-- | app/services/batched_remove_status_service.rb | 98 | ||||
| -rw-r--r-- | app/services/delete_account_service.rb | 150 | ||||
| -rw-r--r-- | app/services/import_service.rb | 7 | ||||
| -rw-r--r-- | app/services/remove_status_service.rb | 2 | ||||
| -rw-r--r-- | app/services/report_service.rb | 3 | ||||
| -rw-r--r-- | app/services/resolve_account_service.rb | 17 | ||||
| -rw-r--r-- | app/services/resolve_url_service.rb | 6 |
9 files changed, 205 insertions, 108 deletions
diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index e5bd0c47c..9d01f5386 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -28,7 +28,7 @@ class ActivityPub::FetchRemoteAccountService < BaseService return unless only_key || verified_webfinger? - ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key) + ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key) rescue Oj::ParseError nil end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 0201c30b3..27b088240 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -28,6 +28,8 @@ class ActivityPub::ProcessAccountService < BaseService update_account process_tags process_attachments + + process_duplicate_accounts! if @options[:verified_webfinger] else raise Mastodon::RaceConditionError end @@ -70,34 +72,42 @@ class ActivityPub::ProcessAccountService < BaseService @account.protocol = :activitypub set_suspension! + set_immediate_protocol_attributes! + set_fetchable_key! unless @account.suspended? && @account.suspension_origin_local? set_immediate_attributes! unless @account.suspended? - set_fetchable_attributes! unless @options[:only_keys] || @account.suspended? + set_fetchable_attributes! unless @options[:only_key] || @account.suspended? @account.save_with_optional_media! end - def set_immediate_attributes! + def set_immediate_protocol_attributes! @account.inbox_url = @json['inbox'] || '' @account.outbox_url = @json['outbox'] || '' @account.shared_inbox_url = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || '' @account.followers_url = @json['followers'] || '' - @account.featured_collection_url = @json['featured'] || '' - @account.devices_url = @json['devices'] || '' @account.url = url || @uri @account.uri = @uri + @account.actor_type = actor_type + end + + def set_immediate_attributes! + @account.featured_collection_url = @json['featured'] || '' + @account.devices_url = @json['devices'] || '' @account.display_name = @json['name'] || '' @account.note = @json['summary'] || '' @account.locked = @json['manuallyApprovesFollowers'] || false @account.fields = property_values || {} @account.also_known_as = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) } - @account.actor_type = actor_type @account.discoverable = @json['discoverable'] || false end + def set_fetchable_key! + @account.public_key = public_key || '' + end + def set_fetchable_attributes! @account.avatar_remote_url = image_url('icon') || '' unless skip_download? @account.header_remote_url = image_url('image') || '' unless skip_download? - @account.public_key = public_key || '' @account.statuses_count = outbox_total_items if outbox_total_items.present? @account.following_count = following_total_items if following_total_items.present? @account.followers_count = followers_total_items if followers_total_items.present? @@ -142,6 +152,12 @@ class ActivityPub::ProcessAccountService < BaseService VerifyAccountLinksWorker.perform_async(@account.id) end + def process_duplicate_accounts! + return unless Account.where(uri: @account.uri).where.not(id: @account.id).exists? + + AccountMergingWorker.perform_async(@account.id) + end + def actor_type if @json['type'].is_a?(Array) @json['type'].find { |type| ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(type) } diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 707672ee0..2b649ee22 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -3,32 +3,41 @@ class BatchedRemoveStatusService < BaseService include Redisable - # Delete given statuses and reblogs of them - # Dispatch PuSH updates of the deleted statuses, but only local ones - # Dispatch Salmon deletes, unique per domain, of the deleted statuses, but only local ones - # Remove statuses from home feeds - # Push delete events to streaming API for home feeds and public feeds - # @param [Enumerable<Status>] statuses A preferably batched array of statuses + # Delete multiple statuses and reblogs of them as efficiently as possible + # @param [Enumerable<Status>] statuses An array of statuses # @param [Hash] options - # @option [Boolean] :skip_side_effects + # @option [Boolean] :skip_side_effects Do not modify feeds and send updates to streaming API def call(statuses, **options) - statuses = Status.where(id: statuses.map(&:id)).includes(:account).flat_map { |status| [status] + status.reblogs.includes(:account).to_a } + ActiveRecord::Associations::Preloader.new.preload(statuses, options[:skip_side_effects] ? :reblogs : [:account, :tags, reblogs: :account]) - @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a } - @tags = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) } + statuses_and_reblogs = statuses.flat_map { |status| [status] + status.reblogs } - @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) } + # The conversations for direct visibility statuses also need + # to be manually updated. This part is not efficient but we + # rely on direct visibility statuses being relatively rare. + statuses_with_account_conversations = statuses.select(&:direct_visibility?) - # Ensure that rendered XML reflects destroyed state - statuses.each do |status| - status.mark_for_mass_destruction! - status.destroy + ActiveRecord::Associations::Preloader.new.preload(statuses_with_account_conversations, [mentions: :account]) + + statuses_with_account_conversations.each do |status| + status.send(:unlink_from_conversations) + unpush_from_direct_timelines(status) end + # We do not batch all deletes into one to avoid having a long-running + # transaction lock the database, but we use the delete method instead + # of destroy to avoid all callbacks. We rely on foreign keys to + # cascade the delete faster without loading the associations. + statuses_and_reblogs.each_slice(50) { |slice| Status.where(id: slice.map(&:id)).delete_all } + + # Since we skipped all callbacks, we also need to manually + # deindex the statuses + Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) if Chewy.enabled? + return if options[:skip_side_effects] # Batch by source account - statuses.group_by(&:account_id).each_value do |account_statuses| + statuses_and_reblogs.group_by(&:account_id).each_value do |account_statuses| account = account_statuses.first.account next unless account @@ -38,20 +47,18 @@ class BatchedRemoveStatusService < BaseService end # Cannot be batched - statuses.each do |status| - unpush_from_public_timelines(status) - unpush_from_direct_timelines(status) if status.direct_visibility? + @status_id_cutoff = Mastodon::Snowflake.id_at(2.weeks.ago) + redis.pipelined do + statuses.each do |status| + unpush_from_public_timelines(status) + end end end private def unpush_from_home_timelines(account, statuses) - recipients = account.followers_for_local_distribution.to_a - - recipients << account if account.local? - - recipients.each do |follower| + account.followers_for_local_distribution.includes(:user).reorder(nil).find_each do |follower| statuses.each do |status| FeedManager.instance.unpush_from_home(follower, status) end @@ -59,7 +66,7 @@ class BatchedRemoveStatusService < BaseService end def unpush_from_list_timelines(account, statuses) - account.lists_for_local_distribution.select(:id, :account_id).each do |list| + account.lists_for_local_distribution.select(:id, :account_id).includes(account: :user).reorder(nil).find_each do |list| statuses.each do |status| FeedManager.instance.unpush_from_list(list, status) end @@ -67,40 +74,27 @@ class BatchedRemoveStatusService < BaseService end def unpush_from_public_timelines(status) - return unless status.public_visibility? + return unless status.public_visibility? && status.id > @status_id_cutoff - payload = @json_payloads[status.id] + payload = Oj.dump(event: :delete, payload: status.id.to_s) - redis.pipelined do - redis.publish('timeline:public', payload) - if status.local? - redis.publish('timeline:public:local', payload) - else - redis.publish('timeline:public:remote', payload) - end - if status.media_attachments.any? - redis.publish('timeline:public:media', payload) - if status.local? - redis.publish('timeline:public:local:media', payload) - else - redis.publish('timeline:public:remote:media', payload) - end - end + redis.publish('timeline:public', payload) + redis.publish(status.local? ? 'timeline:public:local' : 'timeline:public:remote', payload) - @tags[status.id].each do |hashtag| - redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", payload) - redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", payload) if status.local? - end + if status.media_attachments.any? + redis.publish('timeline:public:media', payload) + redis.publish(status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', payload) + end + + status.tags.map { |tag| tag.name.mb_chars.downcase }.each do |hashtag| + redis.publish("timeline:hashtag:#{hashtag}", payload) + redis.publish("timeline:hashtag:#{hashtag}:local", payload) if status.local? end end def unpush_from_direct_timelines(status) - payload = @json_payloads[status.id] - redis.pipelined do - @mentions[status.id].each do |mention| - FeedManager.instance.unpush_from_direct(mention.account, status) if mention.account.local? - end - FeedManager.instance.unpush_from_direct(status.account, status) if status.account.local? + status.mentions.each do |mention| + FeedManager.instance.unpush_from_direct(mention.account, status) if mention.account.local? end end end diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index 9cb80c95a..2bb533cfb 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -6,15 +6,19 @@ class DeleteAccountService < BaseService ASSOCIATIONS_ON_SUSPEND = %w( account_pins active_relationships + aliases block_relationships blocked_by_relationships conversation_mutes conversations custom_filters + devices domain_blocks - favourites + featured_tags follow_requests + identity_proofs list_accounts + migrations mute_relationships muted_by_relationships notifications @@ -25,6 +29,31 @@ class DeleteAccountService < BaseService status_pins ).freeze + # The following associations have no important side-effects + # in callbacks and all of their own associations are secured + # by foreign keys, making them safe to delete without loading + # into memory + ASSOCIATIONS_WITHOUT_SIDE_EFFECTS = %w( + account_pins + aliases + conversation_mutes + conversations + custom_filters + devices + domain_blocks + featured_tags + follow_requests + identity_proofs + list_accounts + migrations + mute_relationships + muted_by_relationships + notifications + owned_lists + scheduled_statuses + status_pins + ) + ASSOCIATIONS_ON_DESTROY = %w( reports targeted_moderation_notes @@ -55,19 +84,25 @@ class DeleteAccountService < BaseService @options[:skip_activitypub] = true if @options[:skip_side_effects] - reject_follows! - undo_follows! - purge_user! - purge_profile! + distribute_activities! purge_content! fulfill_deletion_request! end private - def reject_follows! - return if @account.local? || !@account.activitypub? || @options[:skip_activitypub] + def distribute_activities! + return if skip_activitypub? + + if @account.local? + delete_actor! + elsif @account.activitypub? + reject_follows! + undo_follows! + end + end + def reject_follows! # When deleting a remote account, the account obviously doesn't # actually become deleted on its origin server, i.e. unlike a # locally deleted account it continues to have access to its home @@ -81,8 +116,6 @@ class DeleteAccountService < BaseService end def undo_follows! - return if @account.local? || !@account.activitypub? || @options[:skip_activitypub] - # When deleting a remote account, the account obviously doesn't # actually become deleted on its origin server, but following relationships # are severed on our end. Therefore, make the remote server aware that the @@ -97,7 +130,7 @@ class DeleteAccountService < BaseService def purge_user! return if !@account.local? || @account.user.nil? - if @options[:reserve_email] + if keep_user_record? @account.user.disable! @account.user.invites.where(uses: 0).destroy_all else @@ -106,30 +139,74 @@ class DeleteAccountService < BaseService end def purge_content! - distribute_delete_actor! if @account.local? && !@options[:skip_side_effects] + purge_user! + purge_profile! + purge_statuses! + purge_media_attachments! + purge_polls! + purge_generated_notifications! + purge_favourites! + purge_bookmarks! + purge_feeds! + purge_other_associations! + + @account.destroy unless keep_account_record? + end - @account.statuses.reorder(nil).find_in_batches do |statuses| - statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username] - BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects]) + def purge_statuses! + @account.statuses.reorder(nil).where.not(id: reported_status_ids).in_batches do |statuses| + BatchedRemoveStatusService.new.call(statuses, skip_side_effects: skip_side_effects?) end + end + def purge_media_attachments! @account.media_attachments.reorder(nil).find_each do |media_attachment| - next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id) + next if keep_account_record? && reported_status_ids.include?(media_attachment.status_id) media_attachment.destroy end + end + + def purge_polls! + @account.polls.reorder(nil).where.not(status_id: reported_status_ids).in_batches.delete_all + end + + def purge_generated_notifications! + # By deleting polls and statuses without callbacks, we've left behind + # polymorphically associated notifications generated by this account - @account.polls.reorder(nil).find_each do |poll| - next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id) + Notification.where(from_account: @account).in_batches.delete_all + end + + def purge_favourites! + @account.favourites.in_batches do |favourites| + ids = favourites.pluck(:status_id) + StatusStat.where(status_id: ids).update_all('favourites_count = GREATEST(0, favourites_count - 1)') + Chewy.strategy.current.update(StatusesIndex, ids) if Chewy.enabled? + # Rails.cache.delete_multi would be better, but we don't have it yet + ids.each { |id| Rails.cache.delete("statuses/#{id}") } + favourites.delete_all + end + end - poll.destroy + def purge_bookmarks! + @account.bookmarks.in_batches do |bookmarks| + Chewy.strategy.current.update(StatusesIndex, bookmarks.pluck(:status_id)) if Chewy.enabled? + bookmarks.delete_all end + end + def purge_other_associations! associations_for_destruction.each do |association_name| - destroy_all(@account.public_send(association_name)) + purge_association(association_name) end + end - @account.destroy unless @options[:reserve_username] + def purge_feeds! + return unless @account.local? + + FeedManager.instance.clean_feeds!(:home, [@account.id]) + FeedManager.instance.clean_feeds!(:list, @account.owned_lists.pluck(:id)) end def purge_profile! @@ -137,7 +214,7 @@ class DeleteAccountService < BaseService # there is no point wasting time updating # its values first - return unless @options[:reserve_username] + return unless keep_account_record? @account.silenced_at = nil @account.suspended_at = @options[:suspended_at] || Time.now.utc @@ -152,6 +229,7 @@ class DeleteAccountService < BaseService @account.followers_count = 0 @account.following_count = 0 @account.moved_to_account = nil + @account.also_known_as = [] @account.trust_level = :untrusted @account.avatar.destroy @account.header.destroy @@ -162,11 +240,17 @@ class DeleteAccountService < BaseService @account.deletion_request&.destroy end - def destroy_all(association) - association.in_batches.destroy_all + def purge_association(association_name) + association = @account.public_send(association_name) + + if ASSOCIATIONS_WITHOUT_SIDE_EFFECTS.include?(association_name) + association.in_batches.delete_all + else + association.in_batches.destroy_all + end end - def distribute_delete_actor! + def delete_actor! ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end @@ -193,10 +277,26 @@ class DeleteAccountService < BaseService end def associations_for_destruction - if @options[:reserve_username] + if keep_account_record? ASSOCIATIONS_ON_SUSPEND else ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY end end + + def keep_user_record? + @options[:reserve_email] + end + + def keep_account_record? + @options[:reserve_username] + end + + def skip_side_effects? + @options[:skip_side_effects] + end + + def skip_activitypub? + @options[:skip_activitypub] + end end diff --git a/app/services/import_service.rb b/app/services/import_service.rb index 288e47f1e..0c6ef2238 100644 --- a/app/services/import_service.rb +++ b/app/services/import_service.rb @@ -27,7 +27,7 @@ class ImportService < BaseService def import_follows! parse_import_data!(['Account address']) - import_relationships!('follow', 'unfollow', @account.following, follow_limit, reblogs: { header: 'Show boosts', default: true }) + import_relationships!('follow', 'unfollow', @account.following, ROWS_PROCESSING_LIMIT, reblogs: { header: 'Show boosts', default: true }) end def import_blocks! @@ -85,6 +85,7 @@ class ImportService < BaseService head_items = items.uniq { |acct, _| acct.split('@')[1] } tail_items = items - head_items + Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra| [@account.id, acct, action, extra] end @@ -133,10 +134,6 @@ class ImportService < BaseService Paperclip.io_adapters.for(@import.data).read end - def follow_limit - FollowLimitValidator.limit_for_account(@account) - end - def relations_map_for_account(account, account_ids) { blocking: {}, diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 2918c6f6f..764ed288d 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -160,7 +160,7 @@ class RemoveStatusService < BaseService end def remove_from_direct - @mentions.each do |mention| + @status.active_mentions.each do |mention| FeedManager.instance.unpush_from_direct(mention.account, @status) if mention.account.local? end end diff --git a/app/services/report_service.rb b/app/services/report_service.rb index 1e955c1e7..9d9c7d6c9 100644 --- a/app/services/report_service.rb +++ b/app/services/report_service.rb @@ -24,7 +24,8 @@ class ReportService < BaseService target_account: @target_account, status_ids: @status_ids, comment: @comment, - uri: @options[:uri] + uri: @options[:uri], + forwarded: ActiveModel::Type::Boolean.new.cast(@options[:forward]) ) end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 74b0b82d0..3301aaf51 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -49,7 +49,7 @@ class ResolveAccountService < BaseService # Now it is certain, it is definitely a remote account, and it # either needs to be created, or updated from fresh data - process_account! + fetch_account! rescue Webfinger::Error, Oj::ParseError => e Rails.logger.debug "Webfinger query for #{@uri} failed: #{e}" nil @@ -104,16 +104,12 @@ class ResolveAccountService < BaseService acct.gsub(/\Aacct:/, '').split('@') end - def process_account! + def fetch_account! return unless activitypub_ready? RedisLock.acquire(lock_options) do |lock| if lock.acquired? - @account = Account.find_remote(@username, @domain) - - next if actor_json.nil? - - @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json) + @account = ActivityPub::FetchRemoteAccountService.new.call(actor_url) else raise Mastodon::RaceConditionError end @@ -136,13 +132,6 @@ class ResolveAccountService < BaseService @actor_url ||= @webfinger.link('self', 'href') end - def actor_json - return @actor_json if defined?(@actor_json) - - json = fetch_resource(actor_url, false) - @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil - end - def gone_from_origin? @gone end diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index 2b10ac1e0..5981e4d98 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -34,16 +34,16 @@ class ResolveURLService < BaseService # It may happen that the resource is a private toot, and thus not fetchable, # but we can return the toot if we already know about it. - uris = [@url] + scope = Status.where(uri: @url) # We don't have an index on `url`, so try guessing the `uri` from `url` parsed_url = Addressable::URI.parse(@url) parsed_url.path.match(%r{/@(?<username>#{Account::USERNAME_RE})/(?<status_id>[0-9]+)\Z}) do |matched| parsed_url.path = "/users/#{matched[:username]}/statuses/#{matched[:status_id]}" - uris << parsed_url.to_s + scope = scope.or(Status.where(uri: parsed_url.to_s, url: @url)) end - status = Status.find_by(uri: uris) + status = scope.first authorize_with @on_behalf_of, status, :show? unless status.nil? status |