diff options
author | Starfall <us@starfall.systems> | 2022-07-25 18:53:31 -0500 |
---|---|---|
committer | Starfall <us@starfall.systems> | 2022-07-25 18:53:31 -0500 |
commit | 5b9419060d79eda85c40a12c567dd0e1e44a7ecb (patch) | |
tree | f5e21930844f7c11ae40b9097a78a32916ba5dba /app/lib | |
parent | a137fecf94d25a03ef7224843c1afd0c30f428e6 (diff) | |
parent | 3a7c641dd4db1d67b172f731518b472d58dd2262 (diff) |
Merge remote-tracking branch 'glitch/main'
Diffstat (limited to 'app/lib')
-rw-r--r-- | app/lib/activitypub/parser/media_attachment_parser.rb | 2 | ||||
-rw-r--r-- | app/lib/admin/system_check.rb | 6 | ||||
-rw-r--r-- | app/lib/admin/system_check/base_check.rb | 10 | ||||
-rw-r--r-- | app/lib/admin/system_check/database_schema_check.rb | 4 | ||||
-rw-r--r-- | app/lib/admin/system_check/elasticsearch_check.rb | 8 | ||||
-rw-r--r-- | app/lib/admin/system_check/rules_check.rb | 4 | ||||
-rw-r--r-- | app/lib/admin/system_check/sidekiq_process_check.rb | 4 | ||||
-rw-r--r-- | app/lib/ascii_folding.rb | 10 | ||||
-rw-r--r-- | app/lib/feed_manager.rb | 56 | ||||
-rw-r--r-- | app/lib/hashtag_normalizer.rb | 25 |
10 files changed, 84 insertions, 45 deletions
diff --git a/app/lib/activitypub/parser/media_attachment_parser.rb b/app/lib/activitypub/parser/media_attachment_parser.rb index 30bea1f0e..656be84b7 100644 --- a/app/lib/activitypub/parser/media_attachment_parser.rb +++ b/app/lib/activitypub/parser/media_attachment_parser.rb @@ -50,7 +50,7 @@ class ActivityPub::Parser::MediaAttachmentParser components = begin blurhash = @json['blurhash'] - if blurhash.present? && /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash) + if blurhash.present? && /^[\w#$%*+,-.:;=?@\[\]^{|}~]+$/.match?(blurhash) Blurhash.components(blurhash) end end diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb index 877a42ef6..f512635ab 100644 --- a/app/lib/admin/system_check.rb +++ b/app/lib/admin/system_check.rb @@ -8,11 +8,11 @@ class Admin::SystemCheck Admin::SystemCheck::ElasticsearchCheck, ].freeze - def self.perform + def self.perform(current_user) ACTIVE_CHECKS.each_with_object([]) do |klass, arr| - check = klass.new + check = klass.new(current_user) - if check.pass? + if check.skip? || check.pass? arr else arr << check.message diff --git a/app/lib/admin/system_check/base_check.rb b/app/lib/admin/system_check/base_check.rb index fcad8daca..c2974c218 100644 --- a/app/lib/admin/system_check/base_check.rb +++ b/app/lib/admin/system_check/base_check.rb @@ -1,6 +1,16 @@ # frozen_string_literal: true class Admin::SystemCheck::BaseCheck + attr_reader :current_user + + def initialize(current_user) + @current_user = current_user + end + + def skip? + false + end + def pass? raise NotImplementedError end diff --git a/app/lib/admin/system_check/database_schema_check.rb b/app/lib/admin/system_check/database_schema_check.rb index b93d1954e..c2f01fd55 100644 --- a/app/lib/admin/system_check/database_schema_check.rb +++ b/app/lib/admin/system_check/database_schema_check.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class Admin::SystemCheck::DatabaseSchemaCheck < Admin::SystemCheck::BaseCheck + def skip? + !current_user.can?(:view_devops) + end + def pass? !ActiveRecord::Base.connection.migration_context.needs_migration? end diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 1b48a5415..8aee18267 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck + def skip? + !current_user.can?(:view_devops) + end + def pass? return true unless Chewy.enabled? @@ -32,8 +36,4 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def compatible_version? Gem::Version.new(running_version) >= Gem::Version.new(required_version) end - - def missing_queues - @missing_queues ||= Sidekiq::ProcessSet.new.reduce(SIDEKIQ_QUEUES) { |queues, process| queues - process['queues'] } - end end diff --git a/app/lib/admin/system_check/rules_check.rb b/app/lib/admin/system_check/rules_check.rb index 1fbdf955d..8206a5df3 100644 --- a/app/lib/admin/system_check/rules_check.rb +++ b/app/lib/admin/system_check/rules_check.rb @@ -3,6 +3,10 @@ class Admin::SystemCheck::RulesCheck < Admin::SystemCheck::BaseCheck include RoutingHelper + def skip? + !current_user.can?(:manage_rules) + end + def pass? Rule.kept.exists? end diff --git a/app/lib/admin/system_check/sidekiq_process_check.rb b/app/lib/admin/system_check/sidekiq_process_check.rb index 22446edaf..648811d6c 100644 --- a/app/lib/admin/system_check/sidekiq_process_check.rb +++ b/app/lib/admin/system_check/sidekiq_process_check.rb @@ -9,6 +9,10 @@ class Admin::SystemCheck::SidekiqProcessCheck < Admin::SystemCheck::BaseCheck scheduler ).freeze + def skip? + !current_user.can?(:view_devops) + end + def pass? missing_queues.empty? end diff --git a/app/lib/ascii_folding.rb b/app/lib/ascii_folding.rb new file mode 100644 index 000000000..1798d3d0e --- /dev/null +++ b/app/lib/ascii_folding.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class ASCIIFolding + NON_ASCII_CHARS = 'ÀÁÂÃÄÅàáâãäåĀāĂ㥹ÇçĆćĈĉĊċČčÐðĎďĐđÈÉÊËèéêëĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħÌÍÎÏìíîïĨĩĪīĬĭĮįİıĴĵĶķĸĹĺĻļĽľĿŀŁłÑñŃńŅņŇňʼnŊŋÒÓÔÕÖØòóôõöøŌōŎŏŐőŔŕŖŗŘřŚśŜŝŞşŠšſŢţŤťŦŧÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųŴŵÝýÿŶŷŸŹźŻżŽž' + EQUIVALENT_ASCII_CHARS = 'AAAAAAaaaaaaAaAaAaCcCcCcCcCcDdDdDdEEEEeeeeEeEeEeEeEeGgGgGgGgHhHhIIIIiiiiIiIiIiIiIiJjKkkLlLlLlLlLlNnNnNnNnnNnOOOOOOooooooOoOoOoRrRrRrSsSsSsSssTtTtTtUUUUuuuuUuUuUuUuUuUuWwYyyYyYZzZzZz' + + def fold(str) + str.tr(NON_ASCII_CHARS, EQUIVALENT_ASCII_CHARS) + end +end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 235d6fedd..c607223fc 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -47,6 +47,8 @@ class FeedManager filter_from_mentions?(status, receiver.id) when :direct filter_from_direct?(status, receiver.id) + when :tags + filter_from_tags?(status, receiver.id, build_crutches(receiver.id, [status])) else false end @@ -58,7 +60,7 @@ class FeedManager # @param [Boolean] update # @return [Boolean] def push_to_home(account, status, update: false) - return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?) + return false unless add_to_feed(:home, account.id, status, aggregate_reblogs: account.user&.aggregates_reblogs?) trim(:home, account.id) PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}", { 'update' => update }) if push_update_required?("timeline:#{account.id}") @@ -71,7 +73,7 @@ class FeedManager # @param [Boolean] update # @return [Boolean] def unpush_from_home(account, status, update: false) - return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?) + return false unless remove_from_feed(:home, account.id, status, aggregate_reblogs: account.user&.aggregates_reblogs?) redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update true @@ -83,7 +85,7 @@ class FeedManager # @param [Boolean] update # @return [Boolean] def push_to_list(list, status, update: false) - return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?) trim(:list, list.id) PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", { 'update' => update }) if push_update_required?("timeline:list:#{list.id}") @@ -96,7 +98,7 @@ class FeedManager # @param [Boolean] update # @return [Boolean] def unpush_from_list(list, status, update: false) - return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + return false unless remove_from_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?) redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update true @@ -145,7 +147,7 @@ class FeedManager statuses.each do |status| next if filter_from_home?(status, into_account.id, crutches) - add_to_feed(:home, into_account.id, status, aggregate) + add_to_feed(:home, into_account.id, status, aggregate_reblogs: aggregate) end trim(:home, into_account.id) @@ -171,7 +173,7 @@ class FeedManager statuses.each do |status| next if filter_from_home?(status, list.account_id, crutches) || filter_from_list?(status, list) - add_to_feed(:list, list.id, status, aggregate) + add_to_feed(:list, list.id, status, aggregate_reblogs: aggregate) end trim(:list, list.id) @@ -186,7 +188,7 @@ class FeedManager timeline_status_ids = redis.zrange(timeline_key, 0, -1) from_account.statuses.select('id, reblog_of_id').where(id: timeline_status_ids).reorder(nil).find_each do |status| - remove_from_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?) + remove_from_feed(:home, into_account.id, status, aggregate_reblogs: into_account.user&.aggregates_reblogs?) end end @@ -199,7 +201,7 @@ class FeedManager timeline_status_ids = redis.zrange(timeline_key, 0, -1) from_account.statuses.select('id, reblog_of_id').where(id: timeline_status_ids).reorder(nil).find_each do |status| - remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + remove_from_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?) end end @@ -262,7 +264,7 @@ class FeedManager timeline_key = key(:home, account.id) account.statuses.limit(limit).each do |status| - add_to_feed(:home, account.id, status, aggregate) + add_to_feed(:home, account.id, status, aggregate_reblogs: aggregate) end account.following.includes(:account_stat).find_each do |target_account| @@ -282,7 +284,7 @@ class FeedManager statuses.each do |status| next if filter_from_home?(status, account.id, crutches) - add_to_feed(:home, account.id, status, aggregate) + add_to_feed(:home, account.id, status, aggregate_reblogs: aggregate) end trim(:home, account.id) @@ -401,7 +403,6 @@ class FeedManager def filter_from_home?(status, receiver_id, crutches) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) - return true if phrase_filtered?(status, receiver_id, :home) check_for_blocks = crutches[:active_mentions][status.id] || [] check_for_blocks.concat([status.account_id]) @@ -437,7 +438,6 @@ class FeedManager # @return [Boolean] def filter_from_mentions?(status, receiver_id) return true if receiver_id == status.account_id - return true if phrase_filtered?(status, receiver_id, :notifications) # This filter is called from NotifyService, but already after the sender of # the notification has been checked for mute/block. Therefore, it's not @@ -476,32 +476,14 @@ class FeedManager false end - # Check if the status hits a phrase filter + # Check if a status should not be added to the home feed when it comes + # from a followed hashtag # @param [Status] status # @param [Integer] receiver_id - # @param [Symbol] context + # @param [Hash] crutches # @return [Boolean] - def phrase_filtered?(status, receiver_id, context) - active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a - - active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - - active_filters.map! do |filter| - if filter.whole_word - sb = /\A[[:word:]]/.match?(filter.phrase) ? '\b' : '' - eb = /[[:word:]]\z/.match?(filter.phrase) ? '\b' : '' - - /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ - else - /#{Regexp.escape(filter.phrase)}/i - end - end - - return false if active_filters.empty? - - combined_regex = Regexp.union(active_filters) - - combined_regex.match?(status.proper.searchable_text) + def filter_from_tags?(status, receiver_id, crutches) + receiver_id != status.account_id && (((crutches[:active_mentions][status.id] || []) + [status.account_id]).any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } || crutches[:blocked_by][status.account_id] || crutches[:domain_blocking][status.account.domain]) end # Adds a status to an account's feed, returning true if a status was @@ -513,7 +495,7 @@ class FeedManager # @param [Status] status # @param [Boolean] aggregate_reblogs # @return [Boolean] - def add_to_feed(timeline_type, account_id, status, aggregate_reblogs = true) + def add_to_feed(timeline_type, account_id, status, aggregate_reblogs: true) timeline_key = key(timeline_type, account_id) reblog_key = key(timeline_type, account_id, 'reblogs') @@ -561,7 +543,7 @@ class FeedManager # @param [Status] status # @param [Boolean] aggregate_reblogs # @return [Boolean] - def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs = true) + def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs: true) timeline_key = key(timeline_type, account_id) reblog_key = key(timeline_type, account_id, 'reblogs') diff --git a/app/lib/hashtag_normalizer.rb b/app/lib/hashtag_normalizer.rb new file mode 100644 index 000000000..c1f99e163 --- /dev/null +++ b/app/lib/hashtag_normalizer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class HashtagNormalizer + def normalize(str) + remove_invalid_characters(ascii_folding(lowercase(cjk_width(str)))) + end + + private + + def remove_invalid_characters(str) + str.gsub(/[^[:alnum:]#{Tag::HASHTAG_SEPARATORS}]/, '') + end + + def ascii_folding(str) + ASCIIFolding.new.fold(str) + end + + def lowercase(str) + str.mb_chars.downcase.to_s + end + + def cjk_width(str) + str.unicode_normalize(:nfkc) + end +end |