From e0eb39d41b05115df973d5a9676b67a9309d4ff9 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 2 Nov 2022 16:38:23 +0100 Subject: Fix bookmark import stopping at the first failure (#19669) Fixes #19389 --- app/services/import_service.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/services') diff --git a/app/services/import_service.rb b/app/services/import_service.rb index 676c37bde..ece5b9ef0 100644 --- a/app/services/import_service.rb +++ b/app/services/import_service.rb @@ -112,6 +112,11 @@ class ImportService < BaseService next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri) status || ActivityPub::FetchRemoteStatusService.new.call(uri) + rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError + nil + rescue StandardError => e + Rails.logger.warn "Unexpected error when importing bookmark: #{e}" + nil end account_ids = statuses.map(&:account_id) -- cgit From 4fb0aae636316e79b3c13c4000fda7765fa9474f Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 4 Nov 2022 13:19:12 +0100 Subject: Change mentions of blocked users to not be processed (#19725) Fixes #19698 --- app/services/process_mentions_service.rb | 10 +++ spec/services/process_mentions_service_spec.rb | 100 ++++++++++++++++--------- 2 files changed, 74 insertions(+), 36 deletions(-) (limited to 'app/services') diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index c9c158af1..b117db8c2 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -66,6 +66,16 @@ class ProcessMentionsService < BaseService end def assign_mentions! + # Make sure we never mention blocked accounts + unless @current_mentions.empty? + mentioned_domains = @current_mentions.map { |m| m.account.domain }.compact.uniq + blocked_domains = Set.new(mentioned_domains.empty? ? [] : AccountDomainBlock.where(account_id: @status.account_id, domain: mentioned_domains)) + mentioned_account_ids = @current_mentions.map(&:account_id) + blocked_account_ids = Set.new(@status.account.block_relationships.where(target_account_id: mentioned_account_ids).pluck(:target_account_id)) + + @current_mentions.select! { |mention| !(blocked_account_ids.include?(mention.account_id) || blocked_domains.include?(mention.account.domain)) } + end + @current_mentions.each do |mention| mention.save if mention.new_record? end diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb index 89b265e9a..5b9d17a4c 100644 --- a/spec/services/process_mentions_service_spec.rb +++ b/spec/services/process_mentions_service_spec.rb @@ -1,43 +1,85 @@ require 'rails_helper' RSpec.describe ProcessMentionsService, type: :service do - let(:account) { Fabricate(:account, username: 'alice') } - let(:visibility) { :public } - let(:status) { Fabricate(:status, account: account, text: "Hello @#{remote_user.acct}", visibility: visibility) } + let(:account) { Fabricate(:account, username: 'alice') } subject { ProcessMentionsService.new } - context 'ActivityPub' do - context do - let!(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox') } + context 'when mentions contain blocked accounts' do + let(:non_blocked_account) { Fabricate(:account) } + let(:individually_blocked_account) { Fabricate(:account) } + let(:domain_blocked_account) { Fabricate(:account, domain: 'evil.com') } + let(:status) { Fabricate(:status, account: account, text: "Hello @#{non_blocked_account.acct} @#{individually_blocked_account.acct} @#{domain_blocked_account.acct}", visibility: :public) } - before do - subject.call(status) - end + before do + account.block!(individually_blocked_account) + account.domain_blocks.create!(domain: domain_blocked_account.domain) - it 'creates a mention' do - expect(remote_user.mentions.where(status: status).count).to eq 1 - end + subject.call(status) + end + + it 'creates a mention to the non-blocked account' do + expect(non_blocked_account.mentions.where(status: status).count).to eq 1 end - context 'with an IDN domain' do - let!(:remote_user) { Fabricate(:account, username: 'sneak', protocol: :activitypub, domain: 'xn--hresiar-mxa.ch', inbox_url: 'http://example.com/inbox') } - let!(:status) { Fabricate(:status, account: account, text: "Hello @sneak@hæresiar.ch") } + it 'does not create a mention to the individually blocked account' do + expect(individually_blocked_account.mentions.where(status: status).count).to eq 0 + end - before do - subject.call(status) + it 'does not create a mention to the domain-blocked account' do + expect(domain_blocked_account.mentions.where(status: status).count).to eq 0 + end + end + + context 'resolving a mention to a remote account' do + let(:status) { Fabricate(:status, account: account, text: "Hello @#{remote_user.acct}", visibility: :public) } + + context 'ActivityPub' do + context do + let!(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox') } + + before do + subject.call(status) + end + + it 'creates a mention' do + expect(remote_user.mentions.where(status: status).count).to eq 1 + end end - it 'creates a mention' do - expect(remote_user.mentions.where(status: status).count).to eq 1 + context 'with an IDN domain' do + let!(:remote_user) { Fabricate(:account, username: 'sneak', protocol: :activitypub, domain: 'xn--hresiar-mxa.ch', inbox_url: 'http://example.com/inbox') } + let!(:status) { Fabricate(:status, account: account, text: "Hello @sneak@hæresiar.ch") } + + before do + subject.call(status) + end + + it 'creates a mention' do + expect(remote_user.mentions.where(status: status).count).to eq 1 + end + end + + context 'with an IDN TLD' do + let!(:remote_user) { Fabricate(:account, username: 'foo', protocol: :activitypub, domain: 'xn--y9a3aq.xn--y9a3aq', inbox_url: 'http://example.com/inbox') } + let!(:status) { Fabricate(:status, account: account, text: "Hello @foo@հայ.հայ") } + + before do + subject.call(status) + end + + it 'creates a mention' do + expect(remote_user.mentions.where(status: status).count).to eq 1 + end end end - context 'with an IDN TLD' do - let!(:remote_user) { Fabricate(:account, username: 'foo', protocol: :activitypub, domain: 'xn--y9a3aq.xn--y9a3aq', inbox_url: 'http://example.com/inbox') } - let!(:status) { Fabricate(:status, account: account, text: "Hello @foo@հայ.հայ") } + context 'Temporarily-unreachable ActivityPub user' do + let!(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox', last_webfingered_at: nil) } before do + stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404) + stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:remote_user@example.com").to_return(status: 500) subject.call(status) end @@ -46,18 +88,4 @@ RSpec.describe ProcessMentionsService, type: :service do end end end - - context 'Temporarily-unreachable ActivityPub user' do - let!(:remote_user) { Fabricate(:account, username: 'remote_user', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox', last_webfingered_at: nil) } - - before do - stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404) - stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:remote_user@example.com").to_return(status: 500) - subject.call(status) - end - - it 'creates a mention' do - expect(remote_user.mentions.where(status: status).count).to eq 1 - end - end end -- cgit From 5f9e47be34fcf42ff7fcd5668c7555d4a38e289a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 4 Nov 2022 13:21:06 +0100 Subject: Add caching for payload serialization during fan-out (#19642) --- app/lib/inline_renderer.rb | 11 +++++++ app/lib/status_cache_hydrator.rb | 47 +++++++++++++++++++++++++++ app/services/fan_out_on_write_service.rb | 11 ++++++- app/workers/push_update_worker.rb | 13 ++++---- spec/lib/status_cache_hydrator_spec.rb | 54 ++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 app/lib/status_cache_hydrator.rb create mode 100644 spec/lib/status_cache_hydrator_spec.rb (limited to 'app/services') diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb index b70814748..4bb240b48 100644 --- a/app/lib/inline_renderer.rb +++ b/app/lib/inline_renderer.rb @@ -11,6 +11,7 @@ class InlineRenderer case @template when :status serializer = REST::StatusSerializer + preload_associations_for_status when :notification serializer = REST::NotificationSerializer when :conversation @@ -35,6 +36,16 @@ class InlineRenderer private + def preload_associations_for_status + ActiveRecord::Associations::Preloader.new.preload(@object, { + active_mentions: :account, + + reblog: { + active_mentions: :account, + }, + }) + end + def current_user @current_account&.user end diff --git a/app/lib/status_cache_hydrator.rb b/app/lib/status_cache_hydrator.rb new file mode 100644 index 000000000..01e92b385 --- /dev/null +++ b/app/lib/status_cache_hydrator.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class StatusCacheHydrator + def initialize(status) + @status = status + end + + def hydrate(account_id) + # The cache of the serialized hash is generated by the fan-out-on-write service + payload = Rails.cache.fetch("fan-out/#{@status.id}") { InlineRenderer.render(@status, nil, :status) } + + # If we're delivering to the author who disabled the display of the application used to create the + # status, we need to hydrate the application, since it was not rendered for the basic payload + payload[:application] = ActiveModelSerializers::SerializableResource.new(@status.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json if payload[:application].nil? && @status.account_id == account_id + + # We take advantage of the fact that some relationships can only occur with an original status, not + # the reblog that wraps it, so we can assume that some values are always false + if payload[:reblog] + payload[:favourited] = false + payload[:reblogged] = false + payload[:muted] = false + payload[:bookmarked] = false + payload[:pinned] = false + payload[:filtered] = CustomFilter.apply_cached_filters(CustomFilter.cached_filters_for(@status.reblog_of_id), @status.reblog).map { |filter| ActiveModelSerializers::SerializableResource.new(filter, serializer: REST::FilterResultSerializer).as_json } + + # If the reblogged status is being delivered to the author who disabled the display of the application + # used to create the status, we need to hydrate it here too + payload[:reblog][:application] = ActiveModelSerializers::SerializableResource.new(@status.reblog.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json if payload[:reblog][:application].nil? && @status.reblog.account_id == account_id + + payload[:reblog][:favourited] = Favourite.where(account_id: account_id, status_id: @status.reblog_of_id).exists? + payload[:reblog][:reblogged] = Status.where(account_id: account_id, reblog_of_id: @status.reblog_of_id).exists? + payload[:reblog][:muted] = ConversationMute.where(account_id: account_id, conversation_id: @status.reblog.conversation_id).exists? + payload[:reblog][:bookmarked] = Bookmark.where(account_id: account_id, status_id: @status.reblog_of_id).exists? + payload[:reblog][:pinned] = StatusPin.where(account_id: account_id, status_id: @status.reblog_of_id).exists? + payload[:reblog][:filtered] = payload[:filtered] + else + payload[:favourited] = Favourite.where(account_id: account_id, status_id: @status.id).exists? + payload[:reblogged] = Status.where(account_id: account_id, reblog_of_id: @status.id).exists? + payload[:muted] = ConversationMute.where(account_id: account_id, conversation_id: @status.conversation_id).exists? + payload[:bookmarked] = Bookmark.where(account_id: account_id, status_id: @status.id).exists? + payload[:pinned] = StatusPin.where(account_id: account_id, status_id: @status.id).exists? + payload[:filtered] = CustomFilter.apply_cached_filters(CustomFilter.cached_filters_for(@status.id), @status).map { |filter| ActiveModelSerializers::SerializableResource.new(filter, serializer: REST::FilterResultSerializer).as_json } + end + + payload + end +end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index ce20a146e..2554756a5 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -14,6 +14,7 @@ class FanOutOnWriteService < BaseService @options = options check_race_condition! + warm_payload_cache! fan_out_to_local_recipients! fan_out_to_public_recipients! if broadcastable? @@ -135,13 +136,21 @@ class FanOutOnWriteService < BaseService AccountConversation.add_status(@account, @status) unless update? end + def warm_payload_cache! + Rails.cache.write("fan-out/#{@status.id}", rendered_status) + end + def anonymous_payload @anonymous_payload ||= Oj.dump( event: update? ? :'status.update' : :update, - payload: InlineRenderer.render(@status, nil, :status) + payload: rendered_status ) end + def rendered_status + @rendered_status ||= InlineRenderer.render(@status, nil, :status) + end + def update? @options[:update] end diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 9f44c32b3..72c781749 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -5,11 +5,12 @@ class PushUpdateWorker include Redisable def perform(account_id, status_id, timeline_id = nil, options = {}) - @account = Account.find(account_id) - @status = Status.includes(active_mentions: :account, reblog: { active_mentions: :account }).find(status_id) - @timeline_id = timeline_id || "timeline:#{account.id}" + @status = Status.find(status_id) + @account_id = account_id + @timeline_id = timeline_id || "timeline:#{account_id}" @options = options.symbolize_keys + render_payload! publish! rescue ActiveRecord::RecordNotFound true @@ -17,14 +18,14 @@ class PushUpdateWorker private - def payload - InlineRenderer.render(@status, @account, :status) + def render_payload! + @payload = StatusCacheHydrator.new(@status).hydrate(@account_id) end def message Oj.dump( event: update? ? :'status.update' : :update, - payload: payload, + payload: @payload, queued_at: (Time.now.to_f * 1000.0).to_i ) end diff --git a/spec/lib/status_cache_hydrator_spec.rb b/spec/lib/status_cache_hydrator_spec.rb new file mode 100644 index 000000000..ad9940a85 --- /dev/null +++ b/spec/lib/status_cache_hydrator_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe StatusCacheHydrator do + let(:status) { Fabricate(:status) } + let(:account) { Fabricate(:account) } + + describe '#hydrate' do + subject { described_class.new(status).hydrate(account.id) } + + let(:compare_to_hash) { InlineRenderer.render(status, account, :status) } + + context 'when cache is warm' do + before do + Rails.cache.write("fan-out/#{status.id}", InlineRenderer.render(status, nil, :status)) + end + + it 'renders the same attributes as a full render' do + expect(subject).to include(compare_to_hash) + end + end + + context 'when cache is cold' do + before do + Rails.cache.delete("fan-out/#{status.id}") + end + + it 'renders the same attributes as a full render' do + expect(subject).to include(compare_to_hash) + end + end + + context 'when account has favourited status' do + before do + FavouriteService.new.call(account, status) + end + + it 'renders the same attributes as a full render' do + expect(subject).to include(compare_to_hash) + end + end + + context 'when account has reblogged status' do + before do + ReblogService.new.call(account, status) + end + + it 'renders the same attributes as a full render' do + expect(subject).to include(compare_to_hash) + end + end + end +end -- cgit From c2170991c7889b8f6c6434f416cb0a8450db25a1 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 4 Nov 2022 16:31:44 +0100 Subject: Fix reblogs being discarded after the reblogged status (#19731) --- app/controllers/api/v1/statuses_controller.rb | 2 +- app/lib/status_reach_finder.rb | 2 +- app/models/admin/status_batch_action.rb | 2 +- app/models/status.rb | 6 ++++++ app/services/account_statuses_cleanup_service.rb | 2 +- app/services/remove_status_service.rb | 4 ++-- 6 files changed, 12 insertions(+), 6 deletions(-) (limited to 'app/services') diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 2e239d48b..676ec2a79 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -77,7 +77,7 @@ class Api::V1::StatusesController < Api::BaseController @status = Status.where(account: current_account).find(params[:id]) authorize @status, :destroy? - @status.discard + @status.discard_with_reblogs StatusPin.find_by(status: @status)&.destroy @status.account.statuses_count = @status.account.statuses_count - 1 json = render_to_body json: @status, serializer: REST::StatusSerializer, source_requested: true diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index 98e502bb6..ccf1e9e3a 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -55,7 +55,7 @@ class StatusReachFinder # Beware: Reblogs can be created without the author having had access to the status def reblogs_account_ids - @status.reblogs.pluck(:account_id) if distributable? || unsafe? + @status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).pluck(:account_id) if distributable? || unsafe? end # Beware: Favourites can be created without the author having had access to the status diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb index 0ec4fef82..0f019b854 100644 --- a/app/models/admin/status_batch_action.rb +++ b/app/models/admin/status_batch_action.rb @@ -44,7 +44,7 @@ class Admin::StatusBatchAction ApplicationRecord.transaction do statuses.each do |status| - status.discard + status.discard_with_reblogs log_action(:destroy, status) end diff --git a/app/models/status.rb b/app/models/status.rb index 4805abfea..8bdb5e8db 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -440,6 +440,12 @@ class Status < ApplicationRecord im end + def discard_with_reblogs + discard_time = Time.current + Status.unscoped.where(reblog_of_id: id, deleted_at: [nil, deleted_at]).in_batches.update_all(deleted_at: discard_time) unless reblog? + update_attribute(:deleted_at, discard_time) + end + private def update_status_stat!(attrs) diff --git a/app/services/account_statuses_cleanup_service.rb b/app/services/account_statuses_cleanup_service.rb index 3918b5ba4..96bc3db7d 100644 --- a/app/services/account_statuses_cleanup_service.rb +++ b/app/services/account_statuses_cleanup_service.rb @@ -14,7 +14,7 @@ class AccountStatusesCleanupService < BaseService last_deleted = nil account_policy.statuses_to_delete(budget, cutoff_id, account_policy.last_inspected).reorder(nil).find_each(order: :asc) do |status| - status.discard + status.discard_with_reblogs RemovalWorker.perform_async(status.id, { 'redraft' => false }) num_deleted += 1 last_deleted = status.id diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index f9fdea2cb..37d2dabae 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -19,7 +19,7 @@ class RemoveStatusService < BaseService @options = options with_lock("distribute:#{@status.id}") do - @status.discard + @status.discard_with_reblogs StatusPin.find_by(status: @status)&.destroy @@ -102,7 +102,7 @@ class RemoveStatusService < BaseService # because once original status is gone, reblogs will disappear # without us being able to do all the fancy stuff - @status.reblogs.includes(:account).reorder(nil).find_each do |reblog| + @status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).includes(:account).reorder(nil).find_each do |reblog| RemoveStatusService.new.call(reblog, original_removed: true) end end -- cgit From c4b92b1aee27a813e24395d43e265cc02a8fe9a3 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 5 Nov 2022 00:09:52 +0100 Subject: Fix n+1 query during status removal (#19753) --- app/helpers/application_helper.rb | 2 +- app/services/remove_status_service.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/services') diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 9cc34cab6..8706f5c2a 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -203,7 +203,7 @@ module ApplicationHelper permit_visibilities.shift(permit_visibilities.index(default_privacy) + 1) if default_privacy.present? state_params[:visibility] = params[:visibility] if permit_visibilities.include? params[:visibility] - if user_signed_in? + if user_signed_in? && current_user.functional? state_params[:settings] = state_params[:settings].merge(Web::Setting.find_by(user: current_user)&.data || {}) state_params[:push_subscription] = current_account.user.web_push_subscription(current_session) state_params[:current_account] = current_account diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 37d2dabae..45cfb75f4 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -57,13 +57,13 @@ class RemoveStatusService < BaseService end def remove_from_followers - @account.followers_for_local_distribution.reorder(nil).find_each do |follower| + @account.followers_for_local_distribution.includes(:user).reorder(nil).find_each do |follower| FeedManager.instance.unpush_from_home(follower, @status) end end def remove_from_lists - @account.lists_for_local_distribution.select(:id, :account_id).reorder(nil).find_each do |list| + @account.lists_for_local_distribution.select(:id, :account_id).includes(account: :user).reorder(nil).find_each do |list| FeedManager.instance.unpush_from_list(list, @status) end end -- cgit