diff options
Diffstat (limited to 'app/lib')
| -rw-r--r-- | app/lib/activitypub/activity/announce.rb | 2 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/create.rb | 15 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/delete.rb | 2 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/follow.rb | 2 | ||||
| -rw-r--r-- | app/lib/activitypub/adapter.rb | 1 | ||||
| -rw-r--r-- | app/lib/activitypub/tag_manager.rb | 7 | ||||
| -rw-r--r-- | app/lib/formatter.rb | 2 | ||||
| -rw-r--r-- | app/lib/language_detector.rb | 2 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/base.rb | 71 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/creation.rb | 219 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/deletion.rb | 16 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/general.rb | 20 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/post.rb | 23 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/remote.rb | 11 | ||||
| -rw-r--r-- | app/lib/ostatus/activity/share.rb | 26 | ||||
| -rw-r--r-- | app/lib/ostatus/atom_serializer.rb | 378 | ||||
| -rw-r--r-- | app/lib/request.rb | 6 | ||||
| -rw-r--r-- | app/lib/spam_check.rb | 173 | ||||
| -rw-r--r-- | app/lib/status_finder.rb | 2 | ||||
| -rw-r--r-- | app/lib/tag_manager.rb | 14 | ||||
| -rw-r--r-- | app/lib/webfinger_resource.rb | 6 |
21 files changed, 209 insertions, 789 deletions
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 1aa6ee9ec..34c646668 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -40,7 +40,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity end def announceable?(status) - status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility? + status.account_id == @account.id || status.distributable? end def related_to_local_activity? diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 00f0dd42d..56c24680a 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -41,8 +41,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) + check_for_spam distribute(@status) - forward_for_reply if @status.public_visibility? || @status.unlisted_visibility? + forward_for_reply if @status.distributable? end def find_existing_status @@ -406,6 +407,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Account.local.where(username: local_usernames).exists? end + def check_for_spam + spam_check = SpamCheck.new(@status) + + return if spam_check.skip? + + if spam_check.spam? + spam_check.flag! + else + spam_check.remember! + end + end + def forward_for_reply return unless @json['signature'].present? && reply_to_local? ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 0eb14b89c..1f2b40c15 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -31,7 +31,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity return if @status.nil? - if @status.public_visibility? || @status.unlisted_visibility? + if @status.distributable? forward_for_reply forward_for_reblogs end diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb index 3eb88339a..28f1da19f 100644 --- a/app/lib/activitypub/activity/follow.rb +++ b/app/lib/activitypub/activity/follow.rb @@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account) - if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? + if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor? reject_follow_request!(target_account) return end diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index c259c96f4..a1d84de2f 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -33,6 +33,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base def serializable_hash(options = nil) options = serialization_options(options) serialized_hash = serializer.serializable_hash(options) + serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields] serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options) { '@context' => serialized_context }.merge(serialized_hash) diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 595291342..512272dbe 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -17,7 +17,7 @@ class ActivityPub::TagManager case target.object_type when :person - short_account_url(target) + target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? short_account_status_url(target.account, target) @@ -29,7 +29,7 @@ class ActivityPub::TagManager case target.object_type when :person - account_url(target) + target.instance_actor? ? instance_actor_url : account_url(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? account_status_url(target.account, target) @@ -51,7 +51,7 @@ class ActivityPub::TagManager def replies_uri_for(target, page_params = nil) raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local? - replies_account_status_url(target.account, target, page_params) + account_status_replies_url(target.account, target, page_params) end # Primary audience of a status @@ -119,6 +119,7 @@ class ActivityPub::TagManager def uri_to_local_id(uri, param = :id) path_params = Rails.application.routes.recognize_path(uri) + path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors' path_params[param] end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 4c11ca291..113b5c4a0 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -381,6 +381,6 @@ class Formatter end def mention_html(account) - "<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>" + "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>" end end diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index 1e90af42d..6f9511a54 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -69,7 +69,7 @@ class LanguageDetector new_text = remove_html(text) new_text.gsub!(FetchLinkCardService::URL_PATTERN, '') new_text.gsub!(Account::MENTION_RE, '') - new_text.gsub!(Tag::HASHTAG_RE, '') + new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase } new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '') new_text.gsub!(/\s+/, ' ') new_text diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb deleted file mode 100644 index db70f1998..000000000 --- a/app/lib/ostatus/activity/base.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Base - include Redisable - - def initialize(xml, account = nil, **options) - @xml = xml - @account = account - @options = options - end - - def status? - [:activity, :note, :comment].include?(type) - end - - def verb - raw = @xml.at_xpath('./activity:verb', activity: OStatus::TagManager::AS_XMLNS).content - OStatus::TagManager::VERBS.key(raw) - rescue - :post - end - - def type - raw = @xml.at_xpath('./activity:object-type', activity: OStatus::TagManager::AS_XMLNS).content - OStatus::TagManager::TYPES.key(raw) - rescue - :activity - end - - def id - @xml.at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content - end - - def url - link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| link_candidate['type'] == 'text/html' } - link.nil? ? nil : link['href'] - end - - def activitypub_uri - link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link_candidate['type']) } - link.nil? ? nil : link['href'] - end - - def activitypub_uri? - activitypub_uri.present? - end - - private - - def find_status(uri) - if OStatus::TagManager.instance.local_id?(uri) - local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status') - return Status.find_by(id: local_id) - elsif ActivityPub::TagManager.instance.local_uri?(uri) - local_id = ActivityPub::TagManager.instance.uri_to_local_id(uri) - return Status.find_by(id: local_id) - end - - Status.find_by(uri: uri) - end - - def find_activitypub_status(uri, href) - tag_matches = /tag:([^,:]+)[^:]*:objectId=([\d]+)/.match(uri) - href_matches = %r{/users/([^/]+)}.match(href) - - unless tag_matches.nil? || href_matches.nil? - uri = "https://#{tag_matches[1]}/users/#{href_matches[1]}/statuses/#{tag_matches[2]}" - Status.find_by(uri: uri) - end - end -end diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb deleted file mode 100644 index 60de712db..000000000 --- a/app/lib/ostatus/activity/creation.rb +++ /dev/null @@ -1,219 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Creation < OStatus::Activity::Base - def perform - if redis.exists("delete_upon_arrival:#{@account.id}:#{id}") - Rails.logger.debug "Delete for status #{id} was queued, ignoring" - return [nil, false] - end - - return [nil, false] if @account.suspended? || invalid_origin? - - RedisLock.acquire(lock_options) do |lock| - if lock.acquired? - # Return early if status already exists in db - @status = find_status(id) - return [@status, false] unless @status.nil? - @status = process_status - else - raise Mastodon::RaceConditionError - end - end - - [@status, true] - end - - def process_status - Rails.logger.debug "Creating remote status #{id}" - cached_reblog = reblog - status = nil - - # Skip if the reblogged status is not public - return if cached_reblog && !(cached_reblog.public_visibility? || cached_reblog.unlisted_visibility?) - - media_attachments = save_media.take(4) - - ApplicationRecord.transaction do - status = Status.create!( - uri: id, - url: url, - account: @account, - reblog: cached_reblog, - text: content, - spoiler_text: content_warning, - created_at: published, - override_timestamps: @options[:override_timestamps], - reply: thread?, - language: content_language, - visibility: visibility_scope, - conversation: find_or_create_conversation, - thread: thread? ? find_status(thread.first) || find_activitypub_status(thread.first, thread.second) : nil, - media_attachment_ids: media_attachments.map(&:id), - sensitive: sensitive? - ) - - save_mentions(status) - save_hashtags(status) - save_emojis(status) - end - - if thread? && status.thread.nil? && Request.valid_url?(thread.second) - Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}" - ThreadResolveWorker.perform_async(status.id, thread.second) - end - - Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" - - LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? - - # Only continue if the status is supposed to have arrived in real-time. - # Note that if @options[:override_timestamps] isn't set, the status - # may have a lower snowflake id than other existing statuses, potentially - # "hiding" it from paginated API calls - return status unless @options[:override_timestamps] || status.within_realtime_window? - - DistributionWorker.perform_async(status.id) - - status - end - - def content - @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content - end - - def content_language - @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS)['xml:lang']&.presence || 'en' - end - - def content_warning - @xml.at_xpath('./xmlns:summary', xmlns: OStatus::TagManager::XMLNS)&.content || '' - end - - def visibility_scope - @xml.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content&.to_sym || :public - end - - def published - @xml.at_xpath('./xmlns:published', xmlns: OStatus::TagManager::XMLNS).content - end - - def thread? - !@xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS).nil? - end - - def thread - thr = @xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS) - [thr['ref'], thr['href']] - end - - private - - def sensitive? - # OStatus-specific convention (not standard) - @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).any? { |category| category['term'] == 'nsfw' } - end - - def find_or_create_conversation - uri = @xml.at_xpath('./ostatus:conversation', ostatus: OStatus::TagManager::OS_XMLNS)&.attribute('ref')&.content - return if uri.nil? - - if OStatus::TagManager.instance.local_id?(uri) - local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation') - return Conversation.find_by(id: local_id) - end - - Conversation.find_by(uri: uri) || Conversation.create!(uri: uri) - end - - def save_mentions(parent) - processed_account_ids = [] - - @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each do |link| - next if [OStatus::TagManager::TYPES[:group], OStatus::TagManager::TYPES[:collection]].include? link['ostatus:object-type'] - - mentioned_account = account_from_href(link['href']) - - next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id) - - mentioned_account.mentions.where(status: parent).first_or_create(status: parent) - - # So we can skip duplicate mentions - processed_account_ids << mentioned_account.id - end - end - - def save_hashtags(parent) - tags = @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).map { |category| category['term'] }.select(&:present?) - ProcessHashtagsService.new.call(parent, tags) - end - - def save_media - do_not_download = DomainBlock.reject_media?(@account.domain) - media_attachments = [] - - @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link| - next unless link['href'] - - media = MediaAttachment.where(status: nil, remote_url: link['href']).first_or_initialize(account: @account, status: nil, remote_url: link['href']) - parsed_url = Addressable::URI.parse(link['href']).normalize - - next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? - - media.save - media_attachments << media - - next if do_not_download - - begin - media.file_remote_url = link['href'] - media.save! - rescue ActiveRecord::RecordInvalid - next - end - end - - media_attachments - end - - def save_emojis(parent) - do_not_download = DomainBlock.reject_media?(parent.account.domain) - - return if do_not_download - - @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS).each do |link| - next unless link['href'] && link['name'] - - shortcode = link['name'].delete(':') - emoji = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain) - - next unless emoji.nil? - - emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain) - emoji.image_remote_url = link['href'] - emoji.save - end - end - - def account_from_href(href) - url = Addressable::URI.parse(href).normalize - - if TagManager.instance.web_domain?(url.host) - Account.find_local(url.path.gsub('/users/', '')) - else - Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href) - end - end - - def invalid_origin? - return false unless id.start_with?('http') # Legacy IDs cannot be checked - - needle = Addressable::URI.parse(id).normalized_host - - !(needle.casecmp(@account.domain).zero? || - needle.casecmp(Addressable::URI.parse(@account.remote_url.presence || @account.uri).normalized_host).zero?) - end - - def lock_options - { redis: Redis.current, key: "create:#{id}" } - end -end diff --git a/app/lib/ostatus/activity/deletion.rb b/app/lib/ostatus/activity/deletion.rb deleted file mode 100644 index c98f5ee0a..000000000 --- a/app/lib/ostatus/activity/deletion.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Deletion < OStatus::Activity::Base - def perform - Rails.logger.debug "Deleting remote status #{id}" - - status = Status.find_by(uri: id, account: @account) - status ||= Status.find_by(uri: activitypub_uri, account: @account) if activitypub_uri? - - if status.nil? - redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id) - else - RemoveStatusService.new.call(status) - end - end -end diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb deleted file mode 100644 index 8a6aabc33..000000000 --- a/app/lib/ostatus/activity/general.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::General < OStatus::Activity::Base - def specialize - special_class&.new(@xml, @account, @options) - end - - private - - def special_class - case verb - when :post - OStatus::Activity::Post - when :share - OStatus::Activity::Share - when :delete - OStatus::Activity::Deletion - end - end -end diff --git a/app/lib/ostatus/activity/post.rb b/app/lib/ostatus/activity/post.rb deleted file mode 100644 index 755ed8656..000000000 --- a/app/lib/ostatus/activity/post.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Post < OStatus::Activity::Creation - def perform - status, just_created = super - - if just_created - status.mentions.includes(:account).each do |mention| - mentioned_account = mention.account - next unless mentioned_account.local? - NotifyService.new.call(mentioned_account, mention) - end - end - - status - end - - private - - def reblog - nil - end -end diff --git a/app/lib/ostatus/activity/remote.rb b/app/lib/ostatus/activity/remote.rb deleted file mode 100644 index 5b204b6d8..000000000 --- a/app/lib/ostatus/activity/remote.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Remote < OStatus::Activity::Base - def perform - if activitypub_uri? - find_status(activitypub_uri) || FetchRemoteStatusService.new.call(url) - else - find_status(id) || FetchRemoteStatusService.new.call(url) - end - end -end diff --git a/app/lib/ostatus/activity/share.rb b/app/lib/ostatus/activity/share.rb deleted file mode 100644 index 5ca601415..000000000 --- a/app/lib/ostatus/activity/share.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Share < OStatus::Activity::Creation - def perform - return if reblog.nil? - - status, just_created = super - NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created - status - end - - def object - @xml.at_xpath('.//activity:object', activity: OStatus::TagManager::AS_XMLNS) - end - - private - - def reblog - return @reblog if defined? @reblog - - original_status = OStatus::Activity::Remote.new(object).perform - return if original_status.nil? - - @reblog = original_status.reblog? ? original_status.reblog : original_status - end -end diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb deleted file mode 100644 index 9a05d96cf..000000000 --- a/app/lib/ostatus/atom_serializer.rb +++ /dev/null @@ -1,378 +0,0 @@ -# frozen_string_literal: true - -class OStatus::AtomSerializer - include RoutingHelper - include ActionView::Helpers::SanitizeHelper - - class << self - def render(element) - document = Ox::Document.new(version: '1.0') - document << element - ('<?xml version="1.0"?>' + Ox.dump(element, effort: :tolerant)).force_encoding('UTF-8') - end - end - - def author(account) - author = Ox::Element.new('author') - - uri = OStatus::TagManager.instance.uri_for(account) - - append_element(author, 'id', uri) - append_element(author, 'activity:object-type', OStatus::TagManager::TYPES[:person]) - append_element(author, 'uri', uri) - append_element(author, 'name', account.username) - append_element(author, 'email', account.local? ? account.local_username_and_domain : account.acct) - append_element(author, 'summary', Formatter.instance.simplified_format(account).to_str, type: :html) if account.note? - append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account)) - append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original))) if account.avatar? - append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original))) if account.header? - account.emojis.each do |emoji| - append_element(author, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode) - end - append_element(author, 'poco:preferredUsername', account.username) - append_element(author, 'poco:displayName', account.display_name) if account.display_name? - append_element(author, 'poco:note', account.local? ? account.note : strip_tags(account.note)) if account.note? - append_element(author, 'mastodon:scope', account.locked? ? :private : :public) - - author - end - - def feed(account, stream_entries) - feed = Ox::Element.new('feed') - - add_namespaces(feed) - - append_element(feed, 'id', account_url(account, format: 'atom')) - append_element(feed, 'title', account.display_name.presence || account.username) - append_element(feed, 'subtitle', account.note) - append_element(feed, 'updated', account.updated_at.iso8601) - append_element(feed, 'logo', full_asset_url(account.avatar.url(:original))) - - feed << author(account) - - append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account)) - append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom')) - append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20 - append_element(feed, 'link', nil, rel: :hub, href: api_push_url) - append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id)) - - stream_entries.each do |stream_entry| - feed << entry(stream_entry) - end - - feed - end - - def entry(stream_entry, root = false) - entry = Ox::Element.new('entry') - - add_namespaces(entry) if root - - append_element(entry, 'id', OStatus::TagManager.instance.uri_for(stream_entry.status)) - append_element(entry, 'published', stream_entry.created_at.iso8601) - append_element(entry, 'updated', stream_entry.updated_at.iso8601) - append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status") - - entry << author(stream_entry.account) if root - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[stream_entry.object_type]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[stream_entry.verb]) - - entry << object(stream_entry.target) if stream_entry.targeted? - - if stream_entry.status.nil? - append_element(entry, 'content', 'Deleted status') - elsif stream_entry.status.destroyed? - append_element(entry, 'content', 'Deleted status') - append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(stream_entry.status)) if stream_entry.account.local? - else - serialize_status_attributes(entry, stream_entry.status) - end - - append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(stream_entry.status)) - append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom')) - append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(stream_entry.thread), href: ::TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded? - append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil? - - entry - end - - def object(status) - object = Ox::Element.new('activity:object') - - append_element(object, 'id', OStatus::TagManager.instance.uri_for(status)) - append_element(object, 'published', status.created_at.iso8601) - append_element(object, 'updated', status.updated_at.iso8601) - append_element(object, 'title', status.title) - - object << author(status.account) - - append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[status.object_type]) - append_element(object, 'activity:verb', OStatus::TagManager::VERBS[status.verb]) - - serialize_status_attributes(object, status) - - append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(status)) - append_element(object, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(status.thread), href: ::TagManager.instance.url_for(status.thread)) unless status.thread.nil? - append_element(object, 'ostatus:conversation', nil, ref: conversation_uri(status.conversation)) unless status.conversation_id.nil? - - object - end - - def follow_salmon(follow) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - description = "#{follow.account.acct} started following #{follow.target_account.acct}" - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow')) - append_element(entry, 'title', description) - append_element(entry, 'content', description, type: :html) - - entry << author(follow.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:follow]) - - object = author(follow.target_account) - object.value = 'activity:object' - - entry << object - entry - end - - def follow_request_salmon(follow_request) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest')) - append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}") - - entry << author(follow_request.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:request_friend]) - - object = author(follow_request.target_account) - object.value = 'activity:object' - - entry << object - entry - end - - def authorize_follow_request_salmon(follow_request) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest')) - append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}") - - entry << author(follow_request.target_account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:authorize]) - - object = Ox::Element.new('activity:object') - object << author(follow_request.account) - - append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend]) - - inner_object = author(follow_request.target_account) - inner_object.value = 'activity:object' - - object << inner_object - entry << object - entry - end - - def reject_follow_request_salmon(follow_request) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest')) - append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}") - - entry << author(follow_request.target_account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:reject]) - - object = Ox::Element.new('activity:object') - object << author(follow_request.account) - - append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend]) - - inner_object = author(follow_request.target_account) - inner_object.value = 'activity:object' - - object << inner_object - entry << object - entry - end - - def unfollow_salmon(follow) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}" - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow')) - append_element(entry, 'title', description) - append_element(entry, 'content', description, type: :html) - - entry << author(follow.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfollow]) - - object = author(follow.target_account) - object.value = 'activity:object' - - entry << object - entry - end - - def block_salmon(block) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}" - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block')) - append_element(entry, 'title', description) - - entry << author(block.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:block]) - - object = author(block.target_account) - object.value = 'activity:object' - - entry << object - entry - end - - def unblock_salmon(block) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - description = "#{block.account.acct} no longer blocks #{block.target_account.acct}" - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block')) - append_element(entry, 'title', description) - - entry << author(block.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unblock]) - - object = author(block.target_account) - object.value = 'activity:object' - - entry << object - entry - end - - def favourite_salmon(favourite) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}" - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite')) - append_element(entry, 'title', description) - append_element(entry, 'content', description, type: :html) - - entry << author(favourite.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:favorite]) - - entry << object(favourite.status) - - append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status)) - - entry - end - - def unfavourite_salmon(favourite) - entry = Ox::Element.new('entry') - add_namespaces(entry) - - description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}" - - append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite')) - append_element(entry, 'title', description) - append_element(entry, 'content', description, type: :html) - - entry << author(favourite.account) - - append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity]) - append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfavorite]) - - entry << object(favourite.status) - - append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status)) - - entry - end - - private - - def append_element(parent, name, content = nil, **attributes) - element = Ox::Element.new(name) - attributes.each { |k, v| element[k] = sanitize_str(v) } - element << sanitize_str(content) unless content.nil? - parent << element - end - - def sanitize_str(raw_str) - raw_str.to_s - end - - def conversation_uri(conversation) - return conversation.uri if conversation.uri? - OStatus::TagManager.instance.unique_tag(conversation.created_at, conversation.id, 'Conversation') - end - - def add_namespaces(parent) - parent['xmlns'] = OStatus::TagManager::XMLNS - parent['xmlns:thr'] = OStatus::TagManager::THR_XMLNS - parent['xmlns:activity'] = OStatus::TagManager::AS_XMLNS - parent['xmlns:poco'] = OStatus::TagManager::POCO_XMLNS - parent['xmlns:media'] = OStatus::TagManager::MEDIA_XMLNS - parent['xmlns:ostatus'] = OStatus::TagManager::OS_XMLNS - parent['xmlns:mastodon'] = OStatus::TagManager::MTDN_XMLNS - end - - def serialize_status_attributes(entry, status) - append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(status)) if status.account.local? - - append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text? - append_element(entry, 'content', Formatter.instance.format(status, inline_poll_options: true).to_str || '.', type: 'html', 'xml:lang': status.language) - - status.active_mentions.sort_by(&:id).each do |mentioned| - append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account)) - end - - append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:collection], href: OStatus::TagManager::COLLECTIONS[:public]) if status.public_visibility? - - status.tags.each do |tag| - append_element(entry, 'category', nil, term: tag.name) - end - - status.media_attachments.each do |media| - append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false))) - end - - append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? && status.media_attachments.any? - append_element(entry, 'mastodon:scope', status.visibility) - - status.emojis.each do |emoji| - append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode) - end - end -end diff --git a/app/lib/request.rb b/app/lib/request.rb index 5f7075a3c..9d874fe2c 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -40,8 +40,8 @@ class Request set_digest! if options.key?(:body) end - def on_behalf_of(account, key_id_format = :acct, sign_with: nil) - raise ArgumentError unless account.local? + def on_behalf_of(account, key_id_format = :uri, sign_with: nil) + raise ArgumentError, 'account must not be nil' if account.nil? @account = account @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair @@ -59,7 +59,7 @@ class Request begin response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers)) rescue => e - raise e.class, "#{e.message} on #{@url}", e.backtrace[0] + raise e.class, "#{e.message} on #{@url}", e.backtrace end begin diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb new file mode 100644 index 000000000..0cf1b8790 --- /dev/null +++ b/app/lib/spam_check.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +class SpamCheck + include Redisable + include ActionView::Helpers::TextHelper + + NILSIMSA_COMPARE_THRESHOLD = 95 + NILSIMSA_MIN_SIZE = 10 + EXPIRE_SET_AFTER = 1.week.seconds + + def initialize(status) + @account = status.account + @status = status + end + + def skip? + disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply? + end + + def spam? + if insufficient_data? + false + elsif nilsimsa? + any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } + else + any_other_digest?('md5') { |_, other_digest| other_digest == digest } + end + end + + def flag! + auto_silence_account! + auto_report_status! + end + + def remember! + # The scores in sorted sets don't actually have enough bits to hold an exact + # value of our snowflake IDs, so we use it only for its ordering property. To + # get the correct status ID back, we have to save it in the string value + + redis.zadd(redis_key, @status.id, digest_with_algorithm) + redis.zremrangebyrank(redis_key, '0', '-10') + redis.expire(redis_key, EXPIRE_SET_AFTER) + end + + def reset! + redis.del(redis_key) + end + + def hashable_text + return @hashable_text if defined?(@hashable_text) + + @hashable_text = @status.text + @hashable_text = remove_mentions(@hashable_text) + @hashable_text = strip_tags(@hashable_text) unless @status.local? + @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text) + @hashable_text = remove_whitespace(@hashable_text) + end + + def insufficient_data? + hashable_text.blank? + end + + def digest + @digest ||= begin + if nilsimsa? + Nilsimsa.new(hashable_text).hexdigest + else + Digest::MD5.hexdigest(hashable_text) + end + end + end + + def digest_with_algorithm + if nilsimsa? + ['nilsimsa', digest, @status.id].join(':') + else + ['md5', digest, @status.id].join(':') + end + end + + private + + def disabled? + !Setting.spam_check_enabled + end + + def remove_mentions(text) + return text.gsub(Account::MENTION_RE, '') if @status.local? + + Nokogiri::HTML.fragment(text).tap do |html| + mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) } + + html.traverse do |element| + element.unlink if element.name == 'a' && mentions.include?(element['href']) + end + end.to_s + end + + def normalize_unicode(text) + text.unicode_normalize(:nfkc).downcase + end + + def remove_whitespace(text) + text.gsub(/\s+/, ' ').strip + end + + def auto_silence_account! + @account.silence! + end + + def auto_report_status! + status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable? + ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced')) + end + + def already_flagged? + @account.silenced? + end + + def trusted? + @account.trust_level > Account::TRUST_LEVELS[:untrusted] + end + + def no_unsolicited_mentions? + @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) } + end + + def solicited_reply? + !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists? + end + + def nilsimsa_compare_value(first, second) + first = [first].pack('H*') + second = [second].pack('H*') + bits = 0 + + 0.upto(31) do |i| + bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord + end + + 128 - bits # -128 <= Nilsimsa Compare Value <= 128 + end + + def nilsimsa? + hashable_text.size > NILSIMSA_MIN_SIZE + end + + def other_digests + redis.zrange(redis_key, 0, -1) + end + + def any_other_digest?(filter_algorithm) + other_digests.any? do |record| + algorithm, other_digest, status_id = record.split(':') + + next unless algorithm == filter_algorithm + + yield algorithm, other_digest, status_id + end + end + + def matching_status_ids + if nilsimsa? + other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact + else + other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact + end + end + + def redis_key + @redis_key ||= "spam_check:#{@account.id}" + end +end diff --git a/app/lib/status_finder.rb b/app/lib/status_finder.rb index 4d1aed297..22ced8bf8 100644 --- a/app/lib/status_finder.rb +++ b/app/lib/status_finder.rb @@ -13,8 +13,6 @@ class StatusFinder raise ActiveRecord::RecordNotFound unless TagManager.instance.local_url?(url) case recognized_params[:controller] - when 'stream_entries' - StreamEntry.find(recognized_params[:id]).status when 'statuses' Status.find(recognized_params[:id]) else diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index fb364cb98..c88cf4994 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -24,24 +24,16 @@ class TagManager def same_acct?(canonical, needle) return true if canonical.casecmp(needle).zero? + username, domain = needle.split('@') + local_domain?(domain) && canonical.casecmp(username).zero? end def local_url?(url) uri = Addressable::URI.parse(url).normalize domain = uri.host + (uri.port ? ":#{uri.port}" : '') - TagManager.instance.web_domain?(domain) - end - - def url_for(target) - return target.url if target.respond_to?(:local?) && !target.local? - case target.object_type - when :person - short_account_url(target) - when :note, :comment, :activity - short_account_status_url(target.account, target) - end + TagManager.instance.web_domain?(domain) end end diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb index a54a702a2..22d78874a 100644 --- a/app/lib/webfinger_resource.rb +++ b/app/lib/webfinger_resource.rb @@ -23,11 +23,17 @@ class WebfingerResource def username_from_url if account_show_page? path_params[:username] + elsif instance_actor_page? + Rails.configuration.x.local_domain else raise ActiveRecord::RecordNotFound end end + def instance_actor_page? + path_params[:controller] == 'instance_actors' + end + def account_show_page? path_params[:controller] == 'accounts' && path_params[:action] == 'show' end |