From ae62e5fa533831c936b7bbeb12f5b7605125ce54 Mon Sep 17 00:00:00 2001 From: Kaspar V Date: Wed, 11 Jan 2023 21:57:24 +0100 Subject: Fix/remove calling private method with send in model (#22951) * fix(status): remove send usage for private unlink_from_conversations - make unlink_from_conversations public method - rename unlink_from_conversations to unlink_from_conversations! - fix send call on private method in statuses_vacuum and batched_remove_status_service * fix(feeds_vacuum): replace find_in_batches with in_batches because active record query results should be a little more efficient than itterating with map and each. Postgres can grasp such lists of ids much quicker than ruby can. Will probably make allmost no difference, but cannot hurt either. --- app/lib/vacuum/feeds_vacuum.rb | 8 ++++---- app/lib/vacuum/statuses_vacuum.rb | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) (limited to 'app/lib') diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb index f46bcf75f..fb0b8a847 100644 --- a/app/lib/vacuum/feeds_vacuum.rb +++ b/app/lib/vacuum/feeds_vacuum.rb @@ -9,14 +9,14 @@ class Vacuum::FeedsVacuum private def vacuum_inactive_home_feeds! - inactive_users.select(:id, :account_id).find_in_batches do |users| - feed_manager.clean_feeds!(:home, users.map(&:account_id)) + inactive_users.select(:id, :account_id).in_batches do |users| + feed_manager.clean_feeds!(:home, users.pluck(:account_id)) end end def vacuum_inactive_list_feeds! - inactive_users_lists.select(:id).find_in_batches do |lists| - feed_manager.clean_feeds!(:list, lists.map(&:id)) + inactive_users_lists.select(:id).in_batches do |lists| + feed_manager.clean_feeds!(:list, lists.ids) end end diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb index d1c4e7197..28c087b1c 100644 --- a/app/lib/vacuum/statuses_vacuum.rb +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -19,10 +19,7 @@ class Vacuum::StatusesVacuum # as the search index, must be handled first. statuses.direct_visibility .includes(mentions: :account) - .find_each do |status| - # TODO: replace temporary solution - call of private model method - status.send(:unlink_from_conversations) - end + .find_each(&:unlink_from_conversations!) remove_from_search_index(statuses.ids) if Chewy.enabled? # Foreign keys take care of most associated records for us. -- cgit From 0c689b9d014324aba5b8751dacec4c0fc20b2038 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 11 Jan 2023 21:59:13 +0100 Subject: fix: allow verification when page size exceeds 1MB (using HTML5 parser) (#22879) * fix: allow verification when page size exceeds 1MB Truncates the page after 1MB instead Closes #15316 * switch to HTML5 parser, fix rubocop errors * undo rubocop fixes Co-authored-by: Chris Zubak-Skees --- app/lib/request.rb | 14 ++++++++++---- app/services/verify_link_service.rb | 2 +- spec/lib/request_spec.rb | 5 +++++ spec/services/verify_link_service_spec.rb | 27 +++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) (limited to 'app/lib') diff --git a/app/lib/request.rb b/app/lib/request.rb index b2819c8ed..0508169dc 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -154,9 +154,7 @@ class Request end module ClientLimit - def body_with_limit(limit = 1.megabyte) - raise Mastodon::LengthValidationError if content_length.present? && content_length > limit - + def truncated_body(limit = 1.megabyte) if charset.nil? encoding = Encoding::BINARY else @@ -173,11 +171,19 @@ class Request contents << chunk chunk.clear - raise Mastodon::LengthValidationError if contents.bytesize > limit + break if contents.bytesize > limit end contents end + + def body_with_limit(limit = 1.megabyte) + raise Mastodon::LengthValidationError if content_length.present? && content_length > limit + + contents = truncated_body(limit) + raise Mastodon::LengthValidationError if contents.bytesize > limit + contents + end end if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production? diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb index 7496fe2d5..d049b52d1 100644 --- a/app/services/verify_link_service.rb +++ b/app/services/verify_link_service.rb @@ -26,7 +26,7 @@ class VerifyLinkService < BaseService def link_back_present? return false if @body.blank? - links = Nokogiri::HTML(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]') + links = Nokogiri::HTML5(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]') if links.any? { |link| link['href']&.downcase == @link_back.downcase } true diff --git a/spec/lib/request_spec.rb b/spec/lib/request_spec.rb index 5eccf3201..8539944e2 100644 --- a/spec/lib/request_spec.rb +++ b/spec/lib/request_spec.rb @@ -120,6 +120,11 @@ describe Request do expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError end + it 'truncates large monolithic body' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes }) + expect(subject.perform { |response| response.truncated_body.bytesize }).to be < 2.megabytes + end + it 'uses binary encoding if Content-Type does not tell encoding' do stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' }) expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb index 52ba454cc..391560f1c 100644 --- a/spec/services/verify_link_service_spec.rb +++ b/spec/services/verify_link_service_spec.rb @@ -73,6 +73,33 @@ RSpec.describe VerifyLinkService, type: :service do end end + context 'when a document is truncated but the link back is valid' do + let(:html) do + " + + + + +