Merge remote-tracking branch 'glitch/main'

Not at all sure where the admin UI is going to display English language
names now but OK.

6 files changed, 50 insertions, 127 deletions

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index ad273c20b..cf31b6ff6 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -112,7 +112,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         url: @status_parser.url || @status_parser.uri,
         account: @account,
         text: converted_object_type? ? converted_text : (@status_parser.text || ''),
-        language: @status_parser.language || detected_language,
+        language: @status_parser.language,
         spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''),
         created_at: @status_parser.created_at,
         edited_at: @status_parser.edited_at,
@@ -370,10 +370,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n"))
   end
 
-  def detected_language
-    LanguageDetector.instance.detect(@status_parser.text, @account) if supported_object_type?
-  end
-
   def unsupported_media_type?(mime_type)
     mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type)
   end
diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb
index a6aaf5d21..1cc5f4120 100644
--- a/app/lib/admin/metrics/dimension/languages_dimension.rb
+++ b/app/lib/admin/metrics/dimension/languages_dimension.rb
@@ -20,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
 
     rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
 
-    rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
+    rows.map { |row| { key: row['locale'], human_key: standard_locale_name(row['locale']), value: row['value'].to_s } }
   end
 end
diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
index 1cfa07478..afbc8cde8 100644
--- a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
+++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb
@@ -25,7 +25,7 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi
 
     rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
 
-    rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
+    rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } }
   end
 
   private
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index f2c4beed5..94d149da3 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -32,7 +32,7 @@ class Formatter
   include ActionView::Helpers::TextHelper
 
   def format(status, **options)
-    if status.reblog?
+    if status.respond_to?(:reblog?) && status.reblog?
       prepend_reblog = status.reblog.account.acct
       status         = status.proper
     else
@@ -53,7 +53,7 @@ class Formatter
       return html.html_safe # rubocop:disable Rails/OutputSafety
     end
 
-    linkable_accounts = status.active_mentions.map(&:account)
+    linkable_accounts = status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : []
     linkable_accounts << status.account
 
     html = raw_content
diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
deleted file mode 100644
index 40452eddc..000000000
--- a/app/lib/language_detector.rb
+++ /dev/null
@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-class LanguageDetector
-  include Singleton
-
-  WORDS_THRESHOLD        = 4
-  RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}\p{Thai}]+/m
-
-  def initialize
-    @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
-  end
-
-  def detect(text, account)
-    input_text = prepare_text(text)
-
-    return if input_text.blank?
-
-    detect_language_code(input_text) || default_locale(account)
-  end
-
-  def language_names
-    @language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq
-  end
-
-  private
-
-  def prepare_text(text)
-    simplify_text(text).strip
-  end
-
-  def unreliable_input?(text)
-    !reliable_input?(text)
-  end
-
-  def reliable_input?(text)
-    sufficient_text_length?(text) || language_specific_character_set?(text)
-  end
-
-  def sufficient_text_length?(text)
-    text.split(/\s+/).size >= WORDS_THRESHOLD
-  end
-
-  def language_specific_character_set?(text)
-    words = text.scan(RELIABLE_CHARACTERS_RE)
-
-    if words.present?
-      words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size > 0.3
-    else
-      false
-    end
-  end
-
-  def detect_language_code(text)
-    return if unreliable_input?(text)
-
-    result = @identifier.find_language(text)
-
-    iso6391(result.language.to_s).to_sym if result&.reliable?
-  end
-
-  def iso6391(bcp47)
-    iso639 = bcp47.split('-').first
-
-    # CLD3 returns grandfathered language code for Hebrew
-    return 'he' if iso639 == 'iw'
-
-    ISO_639.find(iso639).alpha2
-  end
-
-  def simplify_text(text)
-    new_text = remove_html(text)
-    new_text.gsub!(FetchLinkCardService::URL_PATTERN, '\1')
-    new_text.gsub!(Account::MENTION_RE, '')
-    new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
-    new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
-    new_text.gsub!(/\s+/, ' ')
-    new_text
-  end
-
-  def new_scrubber
-    scrubber = Rails::Html::PermitScrubber.new
-    scrubber.tags = %w(br p)
-    scrubber
-  end
-
-  def scrubber
-    @scrubber ||= new_scrubber
-  end
-
-  def remove_html(text)
-    text = Loofah.fragment(text).scrub!(scrubber).to_s
-    text.gsub!('<br>', "\n")
-    text.gsub!('</p><p>', "\n\n")
-    text.gsub!(/(^<p>|<\/p>$)/, '')
-    text
-  end
-
-  def default_locale(account)
-    account.user_locale&.to_sym || I18n.default_locale if account.local?
-  end
-end
diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index 56ad0717b..fabbd244d 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -2,6 +2,20 @@
 
 class LinkDetailsExtractor
   include ActionView::Helpers::TagHelper
+  include LanguagesHelper
+
+  # Some publications wrap their JSON-LD data in their <script> tags
+  # in commented-out CDATA blocks, they need to be removed before
+  # attempting to parse JSON
+  CDATA_JUNK_PATTERN = %r{^[\s]*(
+    (/\*[\s]*<!\[CDATA\[[\s]*\*/) # Block comment style opening
+    |
+    (//[\s]*<!\[CDATA\[) # Single-line comment style opening
+    |
+    (/\*[\s]*\]\]>[\s]*\*/) # Block comment style closing
+    |
+    (//[\s]*\]\]>) # Single-line comment style closing
+  )[\s]*$}x
 
   class StructuredData
     SUPPORTED_TYPES = %w(
@@ -61,6 +75,10 @@ class LinkDetailsExtractor
       publisher.dig('logo', 'url')
     end
 
+    def valid?
+      json.present?
+    end
+
     private
 
     def author
@@ -134,11 +152,11 @@ class LinkDetailsExtractor
   end
 
   def title
-    structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first
+    html_entities.decode(structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first)
   end
 
   def description
-    structured_data&.description || opengraph_tag('og:description') || meta_tag('description')
+    html_entities.decode(structured_data&.description || opengraph_tag('og:description') || meta_tag('description'))
   end
 
   def image
@@ -146,11 +164,11 @@ class LinkDetailsExtractor
   end
 
   def canonical_url
-    valid_url_or_nil(opengraph_tag('og:url') || link_tag('canonical'), same_origin_only: true) || @original_url.to_s
+    valid_url_or_nil(link_tag('canonical') || opengraph_tag('og:url'), same_origin_only: true) || @original_url.to_s
   end
 
   def provider_name
-    structured_data&.publisher_name || opengraph_tag('og:site_name')
+    html_entities.decode(structured_data&.publisher_name || opengraph_tag('og:site_name'))
   end
 
   def provider_url
@@ -158,7 +176,7 @@ class LinkDetailsExtractor
   end
 
   def author_name
-    structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username')
+    html_entities.decode(structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username'))
   end
 
   def author_url
@@ -201,14 +219,6 @@ class LinkDetailsExtractor
     nil
   end
 
-  def valid_locale_or_nil(str)
-    return nil if str.blank?
-
-    code,  = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
-    locale = ISO_639.find(code)
-    locale&.alpha2
-  end
-
   def link_tag(name)
     document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
   end
@@ -223,10 +233,24 @@ class LinkDetailsExtractor
 
   def structured_data
     @structured_data ||= begin
-      json_ld = document.xpath('//script[@type="application/ld+json"]').map(&:content).first
-      json_ld.present? ? StructuredData.new(json_ld) : nil
-    rescue Oj::ParseError
-      nil
+      # Some publications have more than one JSON-LD definition on the page,
+      # and some of those definitions aren't valid JSON either, so we have
+      # to loop through here until we find something that is the right type
+      # and doesn't break
+      document.xpath('//script[@type="application/ld+json"]').filter_map do |element|
+        json_ld = element.content&.gsub(CDATA_JUNK_PATTERN, '')
+
+        next if json_ld.blank?
+
+        structured_data = StructuredData.new(html_entities.decode(json_ld))
+
+        next unless structured_data.valid?
+
+        structured_data
+      rescue Oj::ParseError, EncodingError
+        Rails.logger.debug("Invalid JSON-LD in #{@original_url}")
+        next
+      end.first
     end
   end
 
@@ -246,4 +270,8 @@ class LinkDetailsExtractor
       detector.strip_tags = true
     end
   end
+
+  def html_entities
+    @html_entities ||= HTMLEntities.new
+  end
 end