From ad207456d64c76d21c17b26a954b459fe2dc0f54 Mon Sep 17 00:00:00 2001
From: "Renato \"Lond\" Cerqueira"
Date: Thu, 16 Nov 2017 10:51:38 -0200
Subject: Improve language filter (#5724)
* Scrub text of html before detecting language.
* Detect language on statuses coming from activitypub.
* Fix rubocop comments.
* Remove custom emoji from text before language detection
---
app/lib/activitypub/activity/create.rb | 2 +-
app/lib/language_detector.rb | 31 +++++++++++++++++++++++++------
2 files changed, 26 insertions(+), 7 deletions(-)
(limited to 'app/lib')
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 376684c00..66e4f7c5e 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -173,7 +173,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
end
def language_from_content
- return nil unless language_map?
+ return LanguageDetector.instance.detect(text_from_content, @account) unless language_map?
@object['contentMap'].keys.first
end
diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index a42460e10..c6f52f0c7 100644
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -38,12 +38,31 @@ class LanguageDetector
end
def simplify_text(text)
- text.dup.tap do |new_text|
- new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
- new_text.gsub!(Account::MENTION_RE, '')
- new_text.gsub!(Tag::HASHTAG_RE, '')
- new_text.gsub!(/\s+/, ' ')
- end
+ new_text = remove_html(text)
+ new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
+ new_text.gsub!(Account::MENTION_RE, '')
+ new_text.gsub!(Tag::HASHTAG_RE, '')
+ new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
+ new_text.gsub!(/\s+/, ' ')
+ new_text
+ end
+
+ def new_scrubber
+ scrubber = Rails::Html::PermitScrubber.new
+ scrubber.tags = %w(br p)
+ scrubber
+ end
+
+ def scrubber
+ @scrubber ||= new_scrubber
+ end
+
+ def remove_html(text)
+ text = Loofah.fragment(text).scrub!(scrubber).to_s
+ text.gsub!('
', "\n")
+ text.gsub!('
', "\n\n")
+ text.gsub!(/(^
|<\/p>$)/, '')
+ text
end
def default_locale(account)
--
cgit
From 3e90987c8b0c620b6fc104350b8eb16d17d7e80a Mon Sep 17 00:00:00 2001
From: Eugen Rochko
Date: Fri, 17 Nov 2017 02:06:26 +0100
Subject: Fix some rubocop style issues (#5730)
---
app/controllers/application_controller.rb | 2 +-
app/controllers/auth/sessions_controller.rb | 2 +-
app/helpers/admin/filter_helper.rb | 2 +-
app/lib/extractor.rb | 3 ++-
app/models/web/push_subscription.rb | 4 ++--
app/services/batched_remove_status_service.rb | 2 +-
app/services/resolve_remote_account_service.rb | 4 ++--
7 files changed, 10 insertions(+), 9 deletions(-)
(limited to 'app/lib')
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index f41a7f9be..a213302cb 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -104,7 +104,7 @@ class ApplicationController < ActionController::Base
unless uncached_ids.empty?
uncached = klass.where(id: uncached_ids).with_includes.map { |item| [item.id, item] }.to_h
- uncached.values.each do |item|
+ uncached.each_value do |item|
Rails.cache.write(item.cache_key, item)
end
end
diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index 463a183e4..a5acb6c36 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -62,7 +62,7 @@ class Auth::SessionsController < Devise::SessionsController
if user_params[:otp_attempt].present? && session[:otp_user_id]
authenticate_with_two_factor_via_otp(user)
- elsif user && user.valid_password?(user_params[:password])
+ elsif user&.valid_password?(user_params[:password])
prompt_for_two_factor(user)
end
end
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 6a57b3d63..e0fae9d9a 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -18,7 +18,7 @@ module Admin::FilterHelper
def selected?(more_params)
new_url = filtered_url_for(more_params)
- filter_link_class(new_url) == 'selected' ? true : false
+ filter_link_class(new_url) == 'selected'
end
private
diff --git a/app/lib/extractor.rb b/app/lib/extractor.rb
index 957364293..738ec89a0 100644
--- a/app/lib/extractor.rb
+++ b/app/lib/extractor.rb
@@ -5,7 +5,8 @@ module Extractor
module_function
- def extract_mentions_or_lists_with_indices(text) # :yields: username, list_slug, start, end
+ # :yields: username, list_slug, start, end
+ def extract_mentions_or_lists_with_indices(text)
return [] unless text =~ Twitter::Regex[:at_signs]
possible_entries = []
diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb
index da67e7665..a41906227 100644
--- a/app/models/web/push_subscription.rb
+++ b/app/models/web/push_subscription.rb
@@ -24,12 +24,12 @@ class Web::PushSubscription < ApplicationRecord
end
def pushable?(notification)
- data && data.key?('alerts') && data['alerts'][notification.type.to_s]
+ data&.key?('alerts') && data['alerts'][notification.type.to_s]
end
def as_payload
payload = { id: id, endpoint: endpoint }
- payload[:alerts] = data['alerts'] if data && data.key?('alerts')
+ payload[:alerts] = data['alerts'] if data&.key?('alerts')
payload
end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 5d83771c9..676a5d04d 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -26,7 +26,7 @@ class BatchedRemoveStatusService < BaseService
statuses.each(&:destroy)
# Batch by source account
- statuses.group_by(&:account_id).each do |_, account_statuses|
+ statuses.group_by(&:account_id).each_value do |account_statuses|
account = account_statuses.first.account
unpush_from_home_timelines(account, account_statuses)
diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb
index 3d0a36f6c..3293fe40f 100644
--- a/app/services/resolve_remote_account_service.rb
+++ b/app/services/resolve_remote_account_service.rb
@@ -124,11 +124,11 @@ class ResolveRemoteAccountService < BaseService
end
def auto_suspend?
- domain_block && domain_block.suspend?
+ domain_block&.suspend?
end
def auto_silence?
- domain_block && domain_block.silence?
+ domain_block&.silence?
end
def domain_block
--
cgit