From ae62e5fa533831c936b7bbeb12f5b7605125ce54 Mon Sep 17 00:00:00 2001 From: Kaspar V Date: Wed, 11 Jan 2023 21:57:24 +0100 Subject: Fix/remove calling private method with send in model (#22951) * fix(status): remove send usage for private unlink_from_conversations - make unlink_from_conversations public method - rename unlink_from_conversations to unlink_from_conversations! - fix send call on private method in statuses_vacuum and batched_remove_status_service * fix(feeds_vacuum): replace find_in_batches with in_batches because active record query results should be a little more efficient than itterating with map and each. Postgres can grasp such lists of ids much quicker than ruby can. Will probably make allmost no difference, but cannot hurt either. --- app/services/batched_remove_status_service.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'app/services') diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 5000062e4..54e5f10a4 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -19,9 +19,7 @@ class BatchedRemoveStatusService < BaseService ActiveRecord::Associations::Preloader.new.preload(statuses_with_account_conversations, [mentions: :account]) - statuses_with_account_conversations.each do |status| - status.send(:unlink_from_conversations) - end + statuses_with_account_conversations.each(&:unlink_from_conversations!) # We do not batch all deletes into one to avoid having a long-running # transaction lock the database, but we use the delete method instead -- cgit From 0c689b9d014324aba5b8751dacec4c0fc20b2038 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 11 Jan 2023 21:59:13 +0100 Subject: fix: allow verification when page size exceeds 1MB (using HTML5 parser) (#22879) * fix: allow verification when page size exceeds 1MB Truncates the page after 1MB instead Closes #15316 * switch to HTML5 parser, fix rubocop errors * undo rubocop fixes Co-authored-by: Chris Zubak-Skees --- app/lib/request.rb | 14 ++++++++++---- app/services/verify_link_service.rb | 2 +- spec/lib/request_spec.rb | 5 +++++ spec/services/verify_link_service_spec.rb | 27 +++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) (limited to 'app/services') diff --git a/app/lib/request.rb b/app/lib/request.rb index b2819c8ed..0508169dc 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -154,9 +154,7 @@ class Request end module ClientLimit - def body_with_limit(limit = 1.megabyte) - raise Mastodon::LengthValidationError if content_length.present? && content_length > limit - + def truncated_body(limit = 1.megabyte) if charset.nil? encoding = Encoding::BINARY else @@ -173,11 +171,19 @@ class Request contents << chunk chunk.clear - raise Mastodon::LengthValidationError if contents.bytesize > limit + break if contents.bytesize > limit end contents end + + def body_with_limit(limit = 1.megabyte) + raise Mastodon::LengthValidationError if content_length.present? && content_length > limit + + contents = truncated_body(limit) + raise Mastodon::LengthValidationError if contents.bytesize > limit + contents + end end if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production? diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb index 7496fe2d5..d049b52d1 100644 --- a/app/services/verify_link_service.rb +++ b/app/services/verify_link_service.rb @@ -26,7 +26,7 @@ class VerifyLinkService < BaseService def link_back_present? return false if @body.blank? - links = Nokogiri::HTML(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]') + links = Nokogiri::HTML5(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]') if links.any? { |link| link['href']&.downcase == @link_back.downcase } true diff --git a/spec/lib/request_spec.rb b/spec/lib/request_spec.rb index 5eccf3201..8539944e2 100644 --- a/spec/lib/request_spec.rb +++ b/spec/lib/request_spec.rb @@ -120,6 +120,11 @@ describe Request do expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError end + it 'truncates large monolithic body' do + stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes }) + expect(subject.perform { |response| response.truncated_body.bytesize }).to be < 2.megabytes + end + it 'uses binary encoding if Content-Type does not tell encoding' do stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' }) expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb index 52ba454cc..391560f1c 100644 --- a/spec/services/verify_link_service_spec.rb +++ b/spec/services/verify_link_service_spec.rb @@ -73,6 +73,33 @@ RSpec.describe VerifyLinkService, type: :service do end end + context 'when a document is truncated but the link back is valid' do + let(:html) do + " + + + + +