From dca70079b1f01624b5656b2811e046d114f913e0 Mon Sep 17 00:00:00 2001
From: multiple creatures
Date: Tue, 7 May 2019 22:38:43 -0500
Subject: Remove automatic language detection.
---
app/lib/activitypub/activity/create.rb | 2 +-
app/lib/language_detector.rb | 99 ----------------------------------
2 files changed, 1 insertion(+), 100 deletions(-)
delete mode 100644 app/lib/language_detector.rb
(limited to 'app/lib')
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index df735a7bf..5514d9a6e 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -338,7 +338,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
elsif summary_language_map?
@object['summaryMap'].keys.first
elsif supported_object_type?
- LanguageDetector.instance.detect(text_from_content, @account)
+ nil
end
end
diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
deleted file mode 100644
index 1e90af42d..000000000
--- a/app/lib/language_detector.rb
+++ /dev/null
@@ -1,99 +0,0 @@
-# frozen_string_literal: true
-
-class LanguageDetector
- include Singleton
-
- WORDS_THRESHOLD = 4
- RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}]+/m
-
- def initialize
- @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
- end
-
- def detect(text, account)
- input_text = prepare_text(text)
-
- return if input_text.blank?
-
- detect_language_code(input_text) || default_locale(account)
- end
-
- def language_names
- @language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq
- end
-
- private
-
- def prepare_text(text)
- simplify_text(text).strip
- end
-
- def unreliable_input?(text)
- !reliable_input?(text)
- end
-
- def reliable_input?(text)
- sufficient_text_length?(text) || language_specific_character_set?(text)
- end
-
- def sufficient_text_length?(text)
- text.split(/\s+/).size >= WORDS_THRESHOLD
- end
-
- def language_specific_character_set?(text)
- words = text.scan(RELIABLE_CHARACTERS_RE)
-
- if words.present?
- words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3
- else
- false
- end
- end
-
- def detect_language_code(text)
- return if unreliable_input?(text)
- result = @identifier.find_language(text)
- iso6391(result.language.to_s).to_sym if result.reliable?
- end
-
- def iso6391(bcp47)
- iso639 = bcp47.split('-').first
-
- # CLD3 returns grandfathered language code for Hebrew
- return 'he' if iso639 == 'iw'
-
- ISO_639.find(iso639).alpha2
- end
-
- def simplify_text(text)
- new_text = remove_html(text)
- new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
- new_text.gsub!(Account::MENTION_RE, '')
- new_text.gsub!(Tag::HASHTAG_RE, '')
- new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
- new_text.gsub!(/\s+/, ' ')
- new_text
- end
-
- def new_scrubber
- scrubber = Rails::Html::PermitScrubber.new
- scrubber.tags = %w(br p)
- scrubber
- end
-
- def scrubber
- @scrubber ||= new_scrubber
- end
-
- def remove_html(text)
- text = Loofah.fragment(text).scrub!(scrubber).to_s
- text.gsub!('
', "\n")
- text.gsub!('
', "\n\n")
- text.gsub!(/(^
|<\/p>$)/, '')
- text
- end
-
- def default_locale(account)
- account.user_locale&.to_sym || I18n.default_locale if account.local?
- end
-end
--
cgit