fix: allow verification when page size exceeds 1MB (using HTML5 parser) (#22879)

* fix: allow verification when page size exceeds 1MB
Truncates the page after 1MB instead

Closes #15316

* switch to HTML5 parser, fix rubocop errors

* undo rubocop fixes

Co-authored-by: Chris Zubak-Skees <chriszs@gmail.com>

2 files changed, 11 insertions, 5 deletions

diff --git a/app/lib/request.rb b/app/lib/request.rb
index b2819c8ed..0508169dc 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -154,9 +154,7 @@ class Request
   end
 
   module ClientLimit
-    def body_with_limit(limit = 1.megabyte)
-      raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
-
+    def truncated_body(limit = 1.megabyte)
       if charset.nil?
         encoding = Encoding::BINARY
       else
@@ -173,11 +171,19 @@ class Request
         contents << chunk
         chunk.clear
 
-        raise Mastodon::LengthValidationError if contents.bytesize > limit
+        break if contents.bytesize > limit
       end
 
       contents
     end
+
+    def body_with_limit(limit = 1.megabyte)
+      raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
+
+      contents = truncated_body(limit)
+      raise Mastodon::LengthValidationError if contents.bytesize > limit
+      contents
+    end
   end
 
   if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production?
diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb
index 7496fe2d5..d049b52d1 100644
--- a/app/services/verify_link_service.rb
+++ b/app/services/verify_link_service.rb
@@ -26,7 +26,7 @@ class VerifyLinkService < BaseService
   def link_back_present?
     return false if @body.blank?
 
-    links = Nokogiri::HTML(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]')
+    links = Nokogiri::HTML5(@body).xpath('//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]')
 
     if links.any? { |link| link['href']&.downcase == @link_back.downcase }
       true