From e8cc63105fe9e5166182ccea28008d880ca43fd9 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Thu, 15 Dec 2022 23:52:06 +0900 Subject: Don't delivery a reply to domains which are blocked by author (#22117) Co-authored-by: Claire --- app/lib/status_reach_finder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index ccf1e9e3a..36fb0e80f 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -70,7 +70,7 @@ class StatusReachFinder def followers_inboxes if @status.in_reply_to_local_account? && distributable? - @status.account.followers.or(@status.thread.account.followers).inboxes + @status.account.followers.or(@status.thread.account.followers.not_domain_blocked_by_account(@status.account)).inboxes elsif @status.direct_visibility? || @status.limited_visibility? [] else -- cgit From 52540771b0e7f69d2d1e0c21b558976000e807e5 Mon Sep 17 00:00:00 2001 From: s0 Date: Fri, 16 Dec 2022 01:56:48 +1100 Subject: Fix crash in elasticsearch_check.rb (#21006) Nil unwrap causes the admin dashboard to crash/500 when the Chewy client info version number value is nil. This occurs when running another ES-compatible backend such as MeiliSearch. Obviously it would be good for chewy to recognise upstream but at least avoiding the crash would be fine. --- app/lib/admin/system_check/elasticsearch_check.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/lib') diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 8aee18267..a63988224 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -34,6 +34,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck end def compatible_version? + return false if running_version.nil? Gem::Version.new(running_version) >= Gem::Version.new(required_version) end end -- cgit From 673c54f114be78e0588f2bb883f2962dbc7574a7 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 15 Dec 2022 17:04:38 +0100 Subject: Fix inability to use local LibreTranslate without setting ALLOWED_PRIVATE_ADDRESSES (#21926) Fixes #20029 --- app/lib/request.rb | 3 ++- app/lib/translation_service/libre_translate.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/request.rb b/app/lib/request.rb index 96d934a8f..b2819c8ed 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -30,7 +30,8 @@ class Request @verb = verb @url = Addressable::URI.parse(url).normalize @http_client = options.delete(:http_client) - @options = options.merge(socket_class: use_proxy? ? ProxySocket : Socket) + @allow_local = options.delete(:allow_local) + @options = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket) @options = @options.merge(proxy_url) if use_proxy? @headers = {} diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 43576e306..4ebe21e45 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -27,7 +27,7 @@ class TranslationService::LibreTranslate < TranslationService def request(text, source_language, target_language) body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) - req = Request.new(:post, "#{@base_url}/translate", body: body) + req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true) req.add_headers('Content-Type': 'application/json') req end -- cgit From d13702ac06e2a57d05e5938966ea190f33472fb9 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 15 Dec 2022 18:55:55 +0100 Subject: Fix status cache hydration discrepancy (#19879) --- app/lib/status_cache_hydrator.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/status_cache_hydrator.rb b/app/lib/status_cache_hydrator.rb index 298d7851a..a84d25694 100644 --- a/app/lib/status_cache_hydrator.rb +++ b/app/lib/status_cache_hydrator.rb @@ -11,7 +11,7 @@ class StatusCacheHydrator # If we're delivering to the author who disabled the display of the application used to create the # status, we need to hydrate the application, since it was not rendered for the basic payload - payload[:application] = ActiveModelSerializers::SerializableResource.new(@status.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json if payload[:application].nil? && @status.account_id == account_id && @status.application.present? + payload[:application] = @status.application.present? ? ActiveModelSerializers::SerializableResource.new(@status.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json : nil if payload[:application].nil? && @status.account_id == account_id # We take advantage of the fact that some relationships can only occur with an original status, not # the reblog that wraps it, so we can assume that some values are always false @@ -23,7 +23,7 @@ class StatusCacheHydrator # If the reblogged status is being delivered to the author who disabled the display of the application # used to create the status, we need to hydrate it here too - payload[:reblog][:application] = ActiveModelSerializers::SerializableResource.new(@status.reblog.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json if payload[:reblog][:application].nil? && @status.reblog.account_id == account_id && @status.reblog.application.present? + payload[:reblog][:application] = @status.reblog.application.present? ? ActiveModelSerializers::SerializableResource.new(@status.reblog.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json : nil if payload[:reblog][:application].nil? && @status.reblog.account_id == account_id payload[:reblog][:favourited] = Favourite.where(account_id: account_id, status_id: @status.reblog_of_id).exists? payload[:reblog][:reblogged] = Status.where(account_id: account_id, reblog_of_id: @status.reblog_of_id).exists? -- cgit From fdd1facba16db75e425c02807323eb2666688652 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Thu, 5 Jan 2023 21:30:38 +0900 Subject: Fix home TL could contain post from who blocked me (#22849) * Fix home tl contains post from who blocked me * Add test * Fix feed_manager's build_crutches blocked_by was not includes status' owner * Add test for status from I blocked * Fix typo --- app/lib/feed_manager.rb | 3 ++- spec/lib/feed_manager_spec.rb | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 510667558..b9c5bc2cd 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -365,6 +365,7 @@ class FeedManager end return true if check_for_blocks.any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } + return true if crutches[:blocked_by][status.account_id] if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply should_filter = !crutches[:following][status.in_reply_to_account_id] # and I'm not following the person it's a reply to @@ -548,7 +549,7 @@ class FeedManager crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true) crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true) crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.flat_map { |s| [s.account.domain, s.reblog&.account&.domain] }.compact).pluck(:domain).index_with(true) - crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).index_with(true) + crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| [s.account_id, s.reblog&.account_id] }.flatten.compact).pluck(:account_id).index_with(true) crutches end diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index 0f3b05e5a..eb55c3983 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -39,6 +39,18 @@ RSpec.describe FeedManager do expect(FeedManager.instance.filter?(:home, reblog, bob)).to be false end + it 'returns true for post from account who blocked me' do + status = Fabricate(:status, text: 'Hello, World', account: alice) + alice.block!(bob) + expect(FeedManager.instance.filter?(:home, status, bob)).to be true + end + + it 'returns true for post from blocked account' do + status = Fabricate(:status, text: 'Hello, World', account: alice) + bob.block!(alice) + expect(FeedManager.instance.filter?(:home, status, bob)).to be true + end + it 'returns true for reblog by followee of blocked account' do status = Fabricate(:status, text: 'Hello world', account: jeff) reblog = Fabricate(:status, reblog: status, account: alice) -- cgit From 06f979098cec3570231d1eca519bbee9a4754cfe Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Thu, 5 Jan 2023 04:44:31 -0800 Subject: Check OpenSearch compatibility version instead of regular version (#22422) Fixes #18535. --- app/lib/admin/system_check/elasticsearch_check.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index a63988224..7f922978f 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -13,7 +13,14 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def message if running_version.present? - Admin::SystemCheck::Message.new(:elasticsearch_version_check, I18n.t('admin.system_checks.elasticsearch_version_check.version_comparison', running_version: running_version, required_version: required_version)) + Admin::SystemCheck::Message.new( + :elasticsearch_version_check, + I18n.t( + 'admin.system_checks.elasticsearch_version_check.version_comparison', + running_version: running_version, + required_version: required_version + ) + ) else Admin::SystemCheck::Message.new(:elasticsearch_running_check) end @@ -23,7 +30,8 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def running_version @running_version ||= begin - Chewy.client.info['version']['number'] + Chewy.client.info['version']['minimum_wire_compatibility_version'] || + Chewy.client.info['version']['number'] rescue Faraday::ConnectionFailed nil end -- cgit From ae62e5fa533831c936b7bbeb12f5b7605125ce54 Mon Sep 17 00:00:00 2001 From: Kaspar V Date: Wed, 11 Jan 2023 21:57:24 +0100 Subject: Fix/remove calling private method with send in model (#22951) * fix(status): remove send usage for private unlink_from_conversations - make unlink_from_conversations public method - rename unlink_from_conversations to unlink_from_conversations! - fix send call on private method in statuses_vacuum and batched_remove_status_service * fix(feeds_vacuum): replace find_in_batches with in_batches because active record query results should be a little more efficient than itterating with map and each. Postgres can grasp such lists of ids much quicker than ruby can. Will probably make allmost no difference, but cannot hurt either. --- app/lib/vacuum/feeds_vacuum.rb | 8 ++++---- app/lib/vacuum/statuses_vacuum.rb | 5 +---- app/models/status.rb | 28 +++++++++++++-------------- app/services/batched_remove_status_service.rb | 4 +--- 4 files changed, 20 insertions(+), 25 deletions(-) (limited to 'app/lib') diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb index f46bcf75f..fb0b8a847 100644 --- a/app/lib/vacuum/feeds_vacuum.rb +++ b/app/lib/vacuum/feeds_vacuum.rb @@ -9,14 +9,14 @@ class Vacuum::FeedsVacuum private def vacuum_inactive_home_feeds! - inactive_users.select(:id, :account_id).find_in_batches do |users| - feed_manager.clean_feeds!(:home, users.map(&:account_id)) + inactive_users.select(:id, :account_id).in_batches do |users| + feed_manager.clean_feeds!(:home, users.pluck(:account_id)) end end def vacuum_inactive_list_feeds! - inactive_users_lists.select(:id).find_in_batches do |lists| - feed_manager.clean_feeds!(:list, lists.map(&:id)) + inactive_users_lists.select(:id).in_batches do |lists| + feed_manager.clean_feeds!(:list, lists.ids) end end diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb index d1c4e7197..28c087b1c 100644 --- a/app/lib/vacuum/statuses_vacuum.rb +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -19,10 +19,7 @@ class Vacuum::StatusesVacuum # as the search index, must be handled first. statuses.direct_visibility .includes(mentions: :account) - .find_each do |status| - # TODO: replace temporary solution - call of private model method - status.send(:unlink_from_conversations) - end + .find_each(&:unlink_from_conversations!) remove_from_search_index(statuses.ids) if Chewy.enabled? # Foreign keys take care of most associated records for us. diff --git a/app/models/status.rb b/app/models/status.rb index 2fe9f2de0..fa9fb9fad 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -29,7 +29,7 @@ # class Status < ApplicationRecord - before_destroy :unlink_from_conversations + before_destroy :unlink_from_conversations! include Discard::Model include Paginable @@ -309,14 +309,14 @@ class Status < ApplicationRecord after_create_commit :store_uri, if: :local? after_create_commit :update_statistics, if: :local? - around_create Mastodon::Snowflake::Callbacks - before_validation :prepare_contents, if: :local? before_validation :set_reblog before_validation :set_visibility before_validation :set_conversation before_validation :set_local + around_create Mastodon::Snowflake::Callbacks + after_create :set_poll_id class << self @@ -447,6 +447,17 @@ class Status < ApplicationRecord update_attribute(:deleted_at, discard_time) end + def unlink_from_conversations! + return unless direct_visibility? + + inbox_owners = mentioned_accounts.local + inbox_owners += [account] if account.local? + + inbox_owners.each do |inbox_owner| + AccountConversation.remove_status(inbox_owner, self) + end + end + private def update_status_stat!(attrs) @@ -524,15 +535,4 @@ class Status < ApplicationRecord reblog&.decrement_count!(:reblogs_count) if reblog? thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && distributable? end - - def unlink_from_conversations - return unless direct_visibility? - - inbox_owners = mentioned_accounts.local - inbox_owners += [account] if account.local? - - inbox_owners.each do |inbox_owner| - AccountConversation.remove_status(inbox_owner, self) - end - end end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 5000062e4..54e5f10a4 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -19,9 +19,7 @@ class BatchedRemoveStatusService < BaseService ActiveRecord::Associations::Preloader.new.preload(statuses_with_account_conversations, [mentions: :account]) - statuses_with_account_conversations.each do |status| - status.send(:unlink_from_conversations) - end + statuses_with_account_conversations.each(&:unlink_from_conversations!) # We do not batch all deletes into one to avoid having a long-running # transaction lock the database, but we use the delete method instead -- cgit From 0c689b9d014324aba5b8751dacec4c0fc20b2038 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 11 Jan 2023 21:59:13 +0100 Subject: fix: allow verification when page size exceeds 1MB (using HTML5 parser) (#22879) * fix: allow verification when page size exceeds 1MB Truncates the page after 1MB instead Closes #15316 * switch to HTML5 parser, fix rubocop errors * undo rubocop fixes Co-authored-by: Chris Zubak-Skees --- app/lib/request.rb | 14 ++++++++++---- app/services/verify_link_service.rb | 2 +- spec/lib/request_spec.rb | 5 +++++ spec/services/verify_link_service_spec.rb | 27 +++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) (limited to 'app/lib') diff --git a/app/lib/request.rb b/app/lib/request.rb index b2819c8ed..0508169dc 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -154,9 +154,7 @@ class Request end module ClientLimit - def body_with_limit(limit = 1.megabyte) - raise Mastodon::LengthValidationError if content_length.present? && content_length > limit - + def truncated_body(limit = 1.megabyte) if charset.nil? encoding = Encoding::BINARY else @@ -173,11 +171,19 @@ class Request contents << chunk chunk.clear - raise Mastodon::LengthValidationError if contents.bytesize > limit + break if contents.bytesize > limit end contents end + + def body_with_limit(limit = 1.megabyte) + raise Mastodon::LengthValidationError if content_length.present? && content_length > limit + + contents = truncated_body(limit) + raise Mastodon::LengthValidationError if contents.bytesize > limit + contents + end end if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production? diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb index 7496fe2d5..d049b52d1 100644 --- a/app/services/verify_link_service.rb +++ b/app/services/verify_link_service.rb @@ -26,7 +26,7 @@ class VerifyLinkService < BaseService def link_back_present? return false if @body.blank? - links = Nokogiri::HTML(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]') + links = Nokogiri::HTML5(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]') if links.any? { |link| link['href']&.downcase == @link_back.downcase } true diff --git a/spec/lib/request_spec.rb b/spec/lib/request_spec.rb index 5eccf3201..8539944e2 100644 --- a/spec/lib/request_spec.rb +++ b/spec/lib/request_spec.rb @@ -120,6 +120,11 @@ describe Request do expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError end + it 'truncates large monolithic body' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes }) + expect(subject.perform { |response| response.truncated_body.bytesize }).to be < 2.megabytes + end + it 'uses binary encoding if Content-Type does not tell encoding' do stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' }) expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb index 52ba454cc..391560f1c 100644 --- a/spec/services/verify_link_service_spec.rb +++ b/spec/services/verify_link_service_spec.rb @@ -73,6 +73,33 @@ RSpec.describe VerifyLinkService, type: :service do end end + context 'when a document is truncated but the link back is valid' do + let(:html) do + " + + + + +