From ad207456d64c76d21c17b26a954b459fe2dc0f54 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Thu, 16 Nov 2017 10:51:38 -0200 Subject: Improve language filter (#5724) * Scrub text of html before detecting language. * Detect language on statuses coming from activitypub. * Fix rubocop comments. * Remove custom emoji from text before language detection --- app/lib/activitypub/activity/create.rb | 2 +- app/lib/language_detector.rb | 31 +++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 376684c00..66e4f7c5e 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -173,7 +173,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def language_from_content - return nil unless language_map? + return LanguageDetector.instance.detect(text_from_content, @account) unless language_map? @object['contentMap'].keys.first end diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index a42460e10..c6f52f0c7 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -38,12 +38,31 @@ class LanguageDetector end def simplify_text(text) - text.dup.tap do |new_text| - new_text.gsub!(FetchLinkCardService::URL_PATTERN, '') - new_text.gsub!(Account::MENTION_RE, '') - new_text.gsub!(Tag::HASHTAG_RE, '') - new_text.gsub!(/\s+/, ' ') - end + new_text = remove_html(text) + new_text.gsub!(FetchLinkCardService::URL_PATTERN, '') + new_text.gsub!(Account::MENTION_RE, '') + new_text.gsub!(Tag::HASHTAG_RE, '') + new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '') + new_text.gsub!(/\s+/, ' ') + new_text + end + + def new_scrubber + scrubber = Rails::Html::PermitScrubber.new + scrubber.tags = %w(br p) + scrubber + end + + def scrubber + @scrubber ||= new_scrubber + end + + def remove_html(text) + text = Loofah.fragment(text).scrub!(scrubber).to_s + text.gsub!('
', "\n") + text.gsub!('

', "\n\n") + text.gsub!(/(^

|<\/p>$)/, '') + text end def default_locale(account) -- cgit From 3e90987c8b0c620b6fc104350b8eb16d17d7e80a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 17 Nov 2017 02:06:26 +0100 Subject: Fix some rubocop style issues (#5730) --- app/controllers/application_controller.rb | 2 +- app/controllers/auth/sessions_controller.rb | 2 +- app/helpers/admin/filter_helper.rb | 2 +- app/lib/extractor.rb | 3 ++- app/models/web/push_subscription.rb | 4 ++-- app/services/batched_remove_status_service.rb | 2 +- app/services/resolve_remote_account_service.rb | 4 ++-- 7 files changed, 10 insertions(+), 9 deletions(-) (limited to 'app/lib') diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index f41a7f9be..a213302cb 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -104,7 +104,7 @@ class ApplicationController < ActionController::Base unless uncached_ids.empty? uncached = klass.where(id: uncached_ids).with_includes.map { |item| [item.id, item] }.to_h - uncached.values.each do |item| + uncached.each_value do |item| Rails.cache.write(item.cache_key, item) end end diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 463a183e4..a5acb6c36 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -62,7 +62,7 @@ class Auth::SessionsController < Devise::SessionsController if user_params[:otp_attempt].present? && session[:otp_user_id] authenticate_with_two_factor_via_otp(user) - elsif user && user.valid_password?(user_params[:password]) + elsif user&.valid_password?(user_params[:password]) prompt_for_two_factor(user) end end diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb index 6a57b3d63..e0fae9d9a 100644 --- a/app/helpers/admin/filter_helper.rb +++ b/app/helpers/admin/filter_helper.rb @@ -18,7 +18,7 @@ module Admin::FilterHelper def selected?(more_params) new_url = filtered_url_for(more_params) - filter_link_class(new_url) == 'selected' ? true : false + filter_link_class(new_url) == 'selected' end private diff --git a/app/lib/extractor.rb b/app/lib/extractor.rb index 957364293..738ec89a0 100644 --- a/app/lib/extractor.rb +++ b/app/lib/extractor.rb @@ -5,7 +5,8 @@ module Extractor module_function - def extract_mentions_or_lists_with_indices(text) # :yields: username, list_slug, start, end + # :yields: username, list_slug, start, end + def extract_mentions_or_lists_with_indices(text) return [] unless text =~ Twitter::Regex[:at_signs] possible_entries = [] diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb index da67e7665..a41906227 100644 --- a/app/models/web/push_subscription.rb +++ b/app/models/web/push_subscription.rb @@ -24,12 +24,12 @@ class Web::PushSubscription < ApplicationRecord end def pushable?(notification) - data && data.key?('alerts') && data['alerts'][notification.type.to_s] + data&.key?('alerts') && data['alerts'][notification.type.to_s] end def as_payload payload = { id: id, endpoint: endpoint } - payload[:alerts] = data['alerts'] if data && data.key?('alerts') + payload[:alerts] = data['alerts'] if data&.key?('alerts') payload end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 5d83771c9..676a5d04d 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -26,7 +26,7 @@ class BatchedRemoveStatusService < BaseService statuses.each(&:destroy) # Batch by source account - statuses.group_by(&:account_id).each do |_, account_statuses| + statuses.group_by(&:account_id).each_value do |account_statuses| account = account_statuses.first.account unpush_from_home_timelines(account, account_statuses) diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb index 3d0a36f6c..3293fe40f 100644 --- a/app/services/resolve_remote_account_service.rb +++ b/app/services/resolve_remote_account_service.rb @@ -124,11 +124,11 @@ class ResolveRemoteAccountService < BaseService end def auto_suspend? - domain_block && domain_block.suspend? + domain_block&.suspend? end def auto_silence? - domain_block && domain_block.silence? + domain_block&.silence? end def domain_block -- cgit