diff options
Diffstat (limited to 'app/lib')
| -rw-r--r-- | app/lib/activitypub/activity/create.rb | 16 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/delete.rb | 2 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/follow.rb | 2 | ||||
| -rw-r--r-- | app/lib/activitypub/adapter.rb | 1 | ||||
| -rw-r--r-- | app/lib/activitypub/tag_manager.rb | 7 | ||||
| -rw-r--r-- | app/lib/formatter.rb | 4 | ||||
| -rw-r--r-- | app/lib/request.rb | 6 | ||||
| -rw-r--r-- | app/lib/spam_check.rb | 173 | ||||
| -rw-r--r-- | app/lib/status_finder.rb | 2 | ||||
| -rw-r--r-- | app/lib/tag_manager.rb | 14 | ||||
| -rw-r--r-- | app/lib/webfinger_resource.rb | 6 |
11 files changed, 208 insertions, 25 deletions
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index b82bfec21..569868c87 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -81,10 +81,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) - + check_for_spam distribute(@status) return if @options[:imported] - forward_for_reply if @status.public_visibility? || @status.unlisted_visibility? + forward_for_reply if @status.distributable? end def find_existing_status @@ -513,6 +513,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Account.local.where(username: local_usernames).exists? end + def check_for_spam + spam_check = SpamCheck.new(@status) + + return if spam_check.skip? + + if spam_check.spam? + spam_check.flag! + else + spam_check.remember! + end + end + def forward_for_reply return unless @json['signature'].present? && reply_to_local? ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index b39c163f5..5a962cd6a 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -30,7 +30,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity return if @status.nil? - if @status.public_visibility? || @status.unlisted_visibility? + if @status.distributable? forward_for_reply forward_for_reblogs end diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb index 73336fde0..9e94857a7 100644 --- a/app/lib/activitypub/activity/follow.rb +++ b/app/lib/activitypub/activity/follow.rb @@ -9,7 +9,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account) - if (rejecting_unknown? && !known?) || target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? + if (rejecting_unknown? && !known?) || target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor? reject_follow_request!(target_account) return end diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index 1a4502bcd..3e0bbaa6a 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -65,6 +65,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base def serializable_hash(options = nil) options = serialization_options(options) serialized_hash = serializer.serializable_hash(options) + serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields] serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options) { '@context' => serialized_context }.merge(serialized_hash) diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 7d1e46600..22e26d116 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -17,7 +17,7 @@ class ActivityPub::TagManager case target.object_type when :person - short_account_url(target) + target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? short_account_status_url(target.account, target) @@ -29,7 +29,7 @@ class ActivityPub::TagManager case target.object_type when :person - account_url(target) + target.instance_actor? ? instance_actor_url : account_url(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? account_status_url(target.account, target) @@ -51,7 +51,7 @@ class ActivityPub::TagManager def replies_uri_for(target, page_params = nil) raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local? - replies_account_status_url(target.account, target, page_params) + account_status_replies_url(target.account, target, page_params) end # Primary audience of a status @@ -119,6 +119,7 @@ class ActivityPub::TagManager def uri_to_local_id(uri, param = :id) path_params = Rails.application.routes.recognize_path(uri) + path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors' path_params[param] end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 5a1572c05..b81ef8668 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -532,7 +532,7 @@ class Formatter gaps = [] total_offset = 0 - escaped = html.gsub(/<[^>]*>/) do |match| + escaped = html.gsub(/<[^>]*>|&#[0-9]+;/) do |match| total_offset += match.length - 1 end_offset = Regexp.last_match.end(0) gaps << [end_offset - total_offset, total_offset] @@ -604,7 +604,7 @@ class Formatter end def mention_html(account) - "<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>" + "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>" end def anchor_html(text, url) diff --git a/app/lib/request.rb b/app/lib/request.rb index c84daaa69..7a8141c7e 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -33,8 +33,8 @@ class Request set_digest! if options.key?(:body) end - def on_behalf_of(account, key_id_format = :acct, sign_with: nil) - raise ArgumentError unless account.local? + def on_behalf_of(account, key_id_format = :uri, sign_with: nil) + raise ArgumentError, 'account must not be nil' if account.nil? @account = account @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair @@ -52,7 +52,7 @@ class Request begin response = http_client.headers(headers).public_send(@verb, @url.to_s, @options) rescue => e - raise e.class, "#{e.message} on #{@url}", e.backtrace[0] + raise e.class, "#{e.message} on #{@url}", e.backtrace end begin diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb new file mode 100644 index 000000000..0cf1b8790 --- /dev/null +++ b/app/lib/spam_check.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +class SpamCheck + include Redisable + include ActionView::Helpers::TextHelper + + NILSIMSA_COMPARE_THRESHOLD = 95 + NILSIMSA_MIN_SIZE = 10 + EXPIRE_SET_AFTER = 1.week.seconds + + def initialize(status) + @account = status.account + @status = status + end + + def skip? + disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply? + end + + def spam? + if insufficient_data? + false + elsif nilsimsa? + any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } + else + any_other_digest?('md5') { |_, other_digest| other_digest == digest } + end + end + + def flag! + auto_silence_account! + auto_report_status! + end + + def remember! + # The scores in sorted sets don't actually have enough bits to hold an exact + # value of our snowflake IDs, so we use it only for its ordering property. To + # get the correct status ID back, we have to save it in the string value + + redis.zadd(redis_key, @status.id, digest_with_algorithm) + redis.zremrangebyrank(redis_key, '0', '-10') + redis.expire(redis_key, EXPIRE_SET_AFTER) + end + + def reset! + redis.del(redis_key) + end + + def hashable_text + return @hashable_text if defined?(@hashable_text) + + @hashable_text = @status.text + @hashable_text = remove_mentions(@hashable_text) + @hashable_text = strip_tags(@hashable_text) unless @status.local? + @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text) + @hashable_text = remove_whitespace(@hashable_text) + end + + def insufficient_data? + hashable_text.blank? + end + + def digest + @digest ||= begin + if nilsimsa? + Nilsimsa.new(hashable_text).hexdigest + else + Digest::MD5.hexdigest(hashable_text) + end + end + end + + def digest_with_algorithm + if nilsimsa? + ['nilsimsa', digest, @status.id].join(':') + else + ['md5', digest, @status.id].join(':') + end + end + + private + + def disabled? + !Setting.spam_check_enabled + end + + def remove_mentions(text) + return text.gsub(Account::MENTION_RE, '') if @status.local? + + Nokogiri::HTML.fragment(text).tap do |html| + mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) } + + html.traverse do |element| + element.unlink if element.name == 'a' && mentions.include?(element['href']) + end + end.to_s + end + + def normalize_unicode(text) + text.unicode_normalize(:nfkc).downcase + end + + def remove_whitespace(text) + text.gsub(/\s+/, ' ').strip + end + + def auto_silence_account! + @account.silence! + end + + def auto_report_status! + status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable? + ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced')) + end + + def already_flagged? + @account.silenced? + end + + def trusted? + @account.trust_level > Account::TRUST_LEVELS[:untrusted] + end + + def no_unsolicited_mentions? + @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) } + end + + def solicited_reply? + !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists? + end + + def nilsimsa_compare_value(first, second) + first = [first].pack('H*') + second = [second].pack('H*') + bits = 0 + + 0.upto(31) do |i| + bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord + end + + 128 - bits # -128 <= Nilsimsa Compare Value <= 128 + end + + def nilsimsa? + hashable_text.size > NILSIMSA_MIN_SIZE + end + + def other_digests + redis.zrange(redis_key, 0, -1) + end + + def any_other_digest?(filter_algorithm) + other_digests.any? do |record| + algorithm, other_digest, status_id = record.split(':') + + next unless algorithm == filter_algorithm + + yield algorithm, other_digest, status_id + end + end + + def matching_status_ids + if nilsimsa? + other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact + else + other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact + end + end + + def redis_key + @redis_key ||= "spam_check:#{@account.id}" + end +end diff --git a/app/lib/status_finder.rb b/app/lib/status_finder.rb index 4d1aed297..22ced8bf8 100644 --- a/app/lib/status_finder.rb +++ b/app/lib/status_finder.rb @@ -13,8 +13,6 @@ class StatusFinder raise ActiveRecord::RecordNotFound unless TagManager.instance.local_url?(url) case recognized_params[:controller] - when 'stream_entries' - StreamEntry.find(recognized_params[:id]).status when 'statuses' Status.find(recognized_params[:id]) else diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index fb364cb98..c88cf4994 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -24,24 +24,16 @@ class TagManager def same_acct?(canonical, needle) return true if canonical.casecmp(needle).zero? + username, domain = needle.split('@') + local_domain?(domain) && canonical.casecmp(username).zero? end def local_url?(url) uri = Addressable::URI.parse(url).normalize domain = uri.host + (uri.port ? ":#{uri.port}" : '') - TagManager.instance.web_domain?(domain) - end - - def url_for(target) - return target.url if target.respond_to?(:local?) && !target.local? - case target.object_type - when :person - short_account_url(target) - when :note, :comment, :activity - short_account_status_url(target.account, target) - end + TagManager.instance.web_domain?(domain) end end diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb index a54a702a2..22d78874a 100644 --- a/app/lib/webfinger_resource.rb +++ b/app/lib/webfinger_resource.rb @@ -23,11 +23,17 @@ class WebfingerResource def username_from_url if account_show_page? path_params[:username] + elsif instance_actor_page? + Rails.configuration.x.local_domain else raise ActiveRecord::RecordNotFound end end + def instance_actor_page? + path_params[:controller] == 'instance_actors' + end + def account_show_page? path_params[:controller] == 'accounts' && path_params[:action] == 'show' end |