fix: allow verification when page size exceeds 1MB (using HTML5 parser) (#22879)

* fix: allow verification when page size exceeds 1MB
Truncates the page after 1MB instead

Closes #15316

* switch to HTML5 parser, fix rubocop errors

* undo rubocop fixes

Co-authored-by: Chris Zubak-Skees <chriszs@gmail.com>

1 files changed, 5 insertions, 0 deletions

diff --git a/spec/lib/request_spec.rb b/spec/lib/request_spec.rb
index 5eccf3201..8539944e2 100644
--- a/spec/lib/request_spec.rb
+++ b/spec/lib/request_spec.rb
@@ -120,6 +120,11 @@ describe Request do
       expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
     end
 
+    it 'truncates large monolithic body' do
+      stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes })
+      expect(subject.perform { |response| response.truncated_body.bytesize }).to be < 2.megabytes
+    end
+
     it 'uses binary encoding if Content-Type does not tell encoding' do
       stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' })
       expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY