Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- Gemfile.lock
- app/controllers/accounts_controller.rb
- app/controllers/admin/dashboard_controller.rb
- app/controllers/follower_accounts_controller.rb
- app/controllers/following_accounts_controller.rb
- app/controllers/remote_follow_controller.rb
- app/controllers/stream_entries_controller.rb
- app/controllers/tags_controller.rb
- app/javascript/packs/public.js
- app/lib/sanitize_config.rb
- app/models/account.rb
- app/models/form/admin_settings.rb
- app/models/media_attachment.rb
- app/models/stream_entry.rb
- app/models/user.rb
- app/serializers/initial_state_serializer.rb
- app/services/batched_remove_status_service.rb
- app/services/post_status_service.rb
- app/services/process_mentions_service.rb
- app/services/reblog_service.rb
- app/services/remove_status_service.rb
- app/views/admin/settings/edit.html.haml
- config/locales/simple_form.pl.yml
- config/settings.yml
- docker-compose.yml

21 files changed, 209 insertions, 789 deletions

diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 1aa6ee9ec..34c646668 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -40,7 +40,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   end
 
   def announceable?(status)
-    status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility?
+    status.account_id == @account.id || status.distributable?
   end
 
   def related_to_local_activity?
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 00f0dd42d..56c24680a 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -41,8 +41,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
+    check_for_spam
     distribute(@status)
-    forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
+    forward_for_reply if @status.distributable?
   end
 
   def find_existing_status
@@ -406,6 +407,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Account.local.where(username: local_usernames).exists?
   end
 
+  def check_for_spam
+    spam_check = SpamCheck.new(@status)
+
+    return if spam_check.skip?
+
+    if spam_check.spam?
+      spam_check.flag!
+    else
+      spam_check.remember!
+    end
+  end
+
   def forward_for_reply
     return unless @json['signature'].present? && reply_to_local?
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 0eb14b89c..1f2b40c15 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -31,7 +31,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
     return if @status.nil?
 
-    if @status.public_visibility? || @status.unlisted_visibility?
+    if @status.distributable?
       forward_for_reply
       forward_for_reblogs
     end
diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb
index 3eb88339a..28f1da19f 100644
--- a/app/lib/activitypub/activity/follow.rb
+++ b/app/lib/activitypub/activity/follow.rb
@@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
 
     return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
 
-    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved?
+    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
       reject_follow_request!(target_account)
       return
     end
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index c259c96f4..a1d84de2f 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -33,6 +33,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   def serializable_hash(options = nil)
     options         = serialization_options(options)
     serialized_hash = serializer.serializable_hash(options)
+    serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields]
     serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options)
 
     { '@context' => serialized_context }.merge(serialized_hash)
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 595291342..512272dbe 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -17,7 +17,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      short_account_url(target)
+      target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
@@ -29,7 +29,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      account_url(target)
+      target.instance_actor? ? instance_actor_url : account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       account_status_url(target.account, target)
@@ -51,7 +51,7 @@ class ActivityPub::TagManager
   def replies_uri_for(target, page_params = nil)
     raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
 
-    replies_account_status_url(target.account, target, page_params)
+    account_status_replies_url(target.account, target, page_params)
   end
 
   # Primary audience of a status
@@ -119,6 +119,7 @@ class ActivityPub::TagManager
 
   def uri_to_local_id(uri, param = :id)
     path_params = Rails.application.routes.recognize_path(uri)
+    path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors'
     path_params[param]
   end
 
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 4c11ca291..113b5c4a0 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -381,6 +381,6 @@ class Formatter
   end
 
   def mention_html(account)
-    "<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
+    "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
   end
 end
diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index 1e90af42d..6f9511a54 100644
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -69,7 +69,7 @@ class LanguageDetector
     new_text = remove_html(text)
     new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
     new_text.gsub!(Account::MENTION_RE, '')
-    new_text.gsub!(Tag::HASHTAG_RE, '')
+    new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
     new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
     new_text.gsub!(/\s+/, ' ')
     new_text
diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb
deleted file mode 100644
index db70f1998..000000000
--- a/app/lib/ostatus/activity/base.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Base
-  include Redisable
-
-  def initialize(xml, account = nil, **options)
-    @xml     = xml
-    @account = account
-    @options = options
-  end
-
-  def status?
-    [:activity, :note, :comment].include?(type)
-  end
-
-  def verb
-    raw = @xml.at_xpath('./activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::VERBS.key(raw)
-  rescue
-    :post
-  end
-
-  def type
-    raw = @xml.at_xpath('./activity:object-type', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::TYPES.key(raw)
-  rescue
-    :activity
-  end
-
-  def id
-    @xml.at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def url
-    link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| link_candidate['type'] == 'text/html' }
-    link.nil? ? nil : link['href']
-  end
-
-  def activitypub_uri
-    link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link_candidate['type']) }
-    link.nil? ? nil : link['href']
-  end
-
-  def activitypub_uri?
-    activitypub_uri.present?
-  end
-
-  private
-
-  def find_status(uri)
-    if OStatus::TagManager.instance.local_id?(uri)
-      local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status')
-      return Status.find_by(id: local_id)
-    elsif ActivityPub::TagManager.instance.local_uri?(uri)
-      local_id = ActivityPub::TagManager.instance.uri_to_local_id(uri)
-      return Status.find_by(id: local_id)
-    end
-
-    Status.find_by(uri: uri)
-  end
-
-  def find_activitypub_status(uri, href)
-    tag_matches = /tag:([^,:]+)[^:]*:objectId=([\d]+)/.match(uri)
-    href_matches = %r{/users/([^/]+)}.match(href)
-
-    unless tag_matches.nil? || href_matches.nil?
-      uri = "https://#{tag_matches[1]}/users/#{href_matches[1]}/statuses/#{tag_matches[2]}"
-      Status.find_by(uri: uri)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
deleted file mode 100644
index 60de712db..000000000
--- a/app/lib/ostatus/activity/creation.rb
+++ /dev/null
@@ -1,219 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Creation < OStatus::Activity::Base
-  def perform
-    if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
-      Rails.logger.debug "Delete for status #{id} was queued, ignoring"
-      return [nil, false]
-    end
-
-    return [nil, false] if @account.suspended? || invalid_origin?
-
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        # Return early if status already exists in db
-        @status = find_status(id)
-        return [@status, false] unless @status.nil?
-        @status = process_status
-      else
-        raise Mastodon::RaceConditionError
-      end
-    end
-
-    [@status, true]
-  end
-
-  def process_status
-    Rails.logger.debug "Creating remote status #{id}"
-    cached_reblog = reblog
-    status = nil
-
-    # Skip if the reblogged status is not public
-    return if cached_reblog && !(cached_reblog.public_visibility? || cached_reblog.unlisted_visibility?)
-
-    media_attachments = save_media.take(4)
-
-    ApplicationRecord.transaction do
-      status = Status.create!(
-        uri: id,
-        url: url,
-        account: @account,
-        reblog: cached_reblog,
-        text: content,
-        spoiler_text: content_warning,
-        created_at: published,
-        override_timestamps: @options[:override_timestamps],
-        reply: thread?,
-        language: content_language,
-        visibility: visibility_scope,
-        conversation: find_or_create_conversation,
-        thread: thread? ? find_status(thread.first) || find_activitypub_status(thread.first, thread.second) : nil,
-        media_attachment_ids: media_attachments.map(&:id),
-        sensitive: sensitive?
-      )
-
-      save_mentions(status)
-      save_hashtags(status)
-      save_emojis(status)
-    end
-
-    if thread? && status.thread.nil? && Request.valid_url?(thread.second)
-      Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
-      ThreadResolveWorker.perform_async(status.id, thread.second)
-    end
-
-    Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
-
-    LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-
-    # Only continue if the status is supposed to have arrived in real-time.
-    # Note that if @options[:override_timestamps] isn't set, the status
-    # may have a lower snowflake id than other existing statuses, potentially
-    # "hiding" it from paginated API calls
-    return status unless @options[:override_timestamps] || status.within_realtime_window?
-
-    DistributionWorker.perform_async(status.id)
-
-    status
-  end
-
-  def content
-    @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def content_language
-    @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS)['xml:lang']&.presence || 'en'
-  end
-
-  def content_warning
-    @xml.at_xpath('./xmlns:summary', xmlns: OStatus::TagManager::XMLNS)&.content || ''
-  end
-
-  def visibility_scope
-    @xml.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content&.to_sym || :public
-  end
-
-  def published
-    @xml.at_xpath('./xmlns:published', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def thread?
-    !@xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS).nil?
-  end
-
-  def thread
-    thr = @xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS)
-    [thr['ref'], thr['href']]
-  end
-
-  private
-
-  def sensitive?
-    # OStatus-specific convention (not standard)
-    @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).any? { |category| category['term'] == 'nsfw' }
-  end
-
-  def find_or_create_conversation
-    uri = @xml.at_xpath('./ostatus:conversation', ostatus: OStatus::TagManager::OS_XMLNS)&.attribute('ref')&.content
-    return if uri.nil?
-
-    if OStatus::TagManager.instance.local_id?(uri)
-      local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
-      return Conversation.find_by(id: local_id)
-    end
-
-    Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
-  end
-
-  def save_mentions(parent)
-    processed_account_ids = []
-
-    @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next if [OStatus::TagManager::TYPES[:group], OStatus::TagManager::TYPES[:collection]].include? link['ostatus:object-type']
-
-      mentioned_account = account_from_href(link['href'])
-
-      next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
-
-      mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
-
-      # So we can skip duplicate mentions
-      processed_account_ids << mentioned_account.id
-    end
-  end
-
-  def save_hashtags(parent)
-    tags = @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
-    ProcessHashtagsService.new.call(parent, tags)
-  end
-
-  def save_media
-    do_not_download = DomainBlock.reject_media?(@account.domain)
-    media_attachments = []
-
-    @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next unless link['href']
-
-      media = MediaAttachment.where(status: nil, remote_url: link['href']).first_or_initialize(account: @account, status: nil, remote_url: link['href'])
-      parsed_url = Addressable::URI.parse(link['href']).normalize
-
-      next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
-
-      media.save
-      media_attachments << media
-
-      next if do_not_download
-
-      begin
-        media.file_remote_url = link['href']
-        media.save!
-      rescue ActiveRecord::RecordInvalid
-        next
-      end
-    end
-
-    media_attachments
-  end
-
-  def save_emojis(parent)
-    do_not_download = DomainBlock.reject_media?(parent.account.domain)
-
-    return if do_not_download
-
-    @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next unless link['href'] && link['name']
-
-      shortcode = link['name'].delete(':')
-      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
-
-      next unless emoji.nil?
-
-      emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
-      emoji.image_remote_url = link['href']
-      emoji.save
-    end
-  end
-
-  def account_from_href(href)
-    url = Addressable::URI.parse(href).normalize
-
-    if TagManager.instance.web_domain?(url.host)
-      Account.find_local(url.path.gsub('/users/', ''))
-    else
-      Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
-    end
-  end
-
-  def invalid_origin?
-    return false unless id.start_with?('http') # Legacy IDs cannot be checked
-
-    needle = Addressable::URI.parse(id).normalized_host
-
-    !(needle.casecmp(@account.domain).zero? ||
-      needle.casecmp(Addressable::URI.parse(@account.remote_url.presence || @account.uri).normalized_host).zero?)
-  end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{id}" }
-  end
-end
diff --git a/app/lib/ostatus/activity/deletion.rb b/app/lib/ostatus/activity/deletion.rb
deleted file mode 100644
index c98f5ee0a..000000000
--- a/app/lib/ostatus/activity/deletion.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Deletion < OStatus::Activity::Base
-  def perform
-    Rails.logger.debug "Deleting remote status #{id}"
-
-    status   = Status.find_by(uri: id, account: @account)
-    status ||= Status.find_by(uri: activitypub_uri, account: @account) if activitypub_uri?
-
-    if status.nil?
-      redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
-    else
-      RemoveStatusService.new.call(status)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb
deleted file mode 100644
index 8a6aabc33..000000000
--- a/app/lib/ostatus/activity/general.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::General < OStatus::Activity::Base
-  def specialize
-    special_class&.new(@xml, @account, @options)
-  end
-
-  private
-
-  def special_class
-    case verb
-    when :post
-      OStatus::Activity::Post
-    when :share
-      OStatus::Activity::Share
-    when :delete
-      OStatus::Activity::Deletion
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/post.rb b/app/lib/ostatus/activity/post.rb
deleted file mode 100644
index 755ed8656..000000000
--- a/app/lib/ostatus/activity/post.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Post < OStatus::Activity::Creation
-  def perform
-    status, just_created = super
-
-    if just_created
-      status.mentions.includes(:account).each do |mention|
-        mentioned_account = mention.account
-        next unless mentioned_account.local?
-        NotifyService.new.call(mentioned_account, mention)
-      end
-    end
-
-    status
-  end
-
-  private
-
-  def reblog
-    nil
-  end
-end
diff --git a/app/lib/ostatus/activity/remote.rb b/app/lib/ostatus/activity/remote.rb
deleted file mode 100644
index 5b204b6d8..000000000
--- a/app/lib/ostatus/activity/remote.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Remote < OStatus::Activity::Base
-  def perform
-    if activitypub_uri?
-      find_status(activitypub_uri) || FetchRemoteStatusService.new.call(url)
-    else
-      find_status(id) || FetchRemoteStatusService.new.call(url)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/share.rb b/app/lib/ostatus/activity/share.rb
deleted file mode 100644
index 5ca601415..000000000
--- a/app/lib/ostatus/activity/share.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Share < OStatus::Activity::Creation
-  def perform
-    return if reblog.nil?
-
-    status, just_created = super
-    NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created
-    status
-  end
-
-  def object
-    @xml.at_xpath('.//activity:object', activity: OStatus::TagManager::AS_XMLNS)
-  end
-
-  private
-
-  def reblog
-    return @reblog if defined? @reblog
-
-    original_status = OStatus::Activity::Remote.new(object).perform
-    return if original_status.nil?
-
-    @reblog = original_status.reblog? ? original_status.reblog : original_status
-  end
-end
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
deleted file mode 100644
index 9a05d96cf..000000000
--- a/app/lib/ostatus/atom_serializer.rb
+++ /dev/null
@@ -1,378 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::AtomSerializer
-  include RoutingHelper
-  include ActionView::Helpers::SanitizeHelper
-
-  class << self
-    def render(element)
-      document = Ox::Document.new(version: '1.0')
-      document << element
-      ('<?xml version="1.0"?>' + Ox.dump(element, effort: :tolerant)).force_encoding('UTF-8')
-    end
-  end
-
-  def author(account)
-    author = Ox::Element.new('author')
-
-    uri = OStatus::TagManager.instance.uri_for(account)
-
-    append_element(author, 'id', uri)
-    append_element(author, 'activity:object-type', OStatus::TagManager::TYPES[:person])
-    append_element(author, 'uri', uri)
-    append_element(author, 'name', account.username)
-    append_element(author, 'email', account.local? ? account.local_username_and_domain : account.acct)
-    append_element(author, 'summary', Formatter.instance.simplified_format(account).to_str, type: :html) if account.note?
-    append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
-    append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original))) if account.avatar?
-    append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original))) if account.header?
-    account.emojis.each do |emoji|
-      append_element(author, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
-    end
-    append_element(author, 'poco:preferredUsername', account.username)
-    append_element(author, 'poco:displayName', account.display_name) if account.display_name?
-    append_element(author, 'poco:note', account.local? ? account.note : strip_tags(account.note)) if account.note?
-    append_element(author, 'mastodon:scope', account.locked? ? :private : :public)
-
-    author
-  end
-
-  def feed(account, stream_entries)
-    feed = Ox::Element.new('feed')
-
-    add_namespaces(feed)
-
-    append_element(feed, 'id', account_url(account, format: 'atom'))
-    append_element(feed, 'title', account.display_name.presence || account.username)
-    append_element(feed, 'subtitle', account.note)
-    append_element(feed, 'updated', account.updated_at.iso8601)
-    append_element(feed, 'logo', full_asset_url(account.avatar.url(:original)))
-
-    feed << author(account)
-
-    append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
-    append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
-    append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
-    append_element(feed, 'link', nil, rel: :hub, href: api_push_url)
-    append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id))
-
-    stream_entries.each do |stream_entry|
-      feed << entry(stream_entry)
-    end
-
-    feed
-  end
-
-  def entry(stream_entry, root = false)
-    entry = Ox::Element.new('entry')
-
-    add_namespaces(entry) if root
-
-    append_element(entry, 'id', OStatus::TagManager.instance.uri_for(stream_entry.status))
-    append_element(entry, 'published', stream_entry.created_at.iso8601)
-    append_element(entry, 'updated', stream_entry.updated_at.iso8601)
-    append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status")
-
-    entry << author(stream_entry.account) if root
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[stream_entry.object_type])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[stream_entry.verb])
-
-    entry << object(stream_entry.target) if stream_entry.targeted?
-
-    if stream_entry.status.nil?
-      append_element(entry, 'content', 'Deleted status')
-    elsif stream_entry.status.destroyed?
-      append_element(entry, 'content', 'Deleted status')
-      append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(stream_entry.status)) if stream_entry.account.local?
-    else
-      serialize_status_attributes(entry, stream_entry.status)
-    end
-
-    append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(stream_entry.status))
-    append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(stream_entry.thread), href: ::TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
-    append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil?
-
-    entry
-  end
-
-  def object(status)
-    object = Ox::Element.new('activity:object')
-
-    append_element(object, 'id', OStatus::TagManager.instance.uri_for(status))
-    append_element(object, 'published', status.created_at.iso8601)
-    append_element(object, 'updated', status.updated_at.iso8601)
-    append_element(object, 'title', status.title)
-
-    object << author(status.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[status.object_type])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[status.verb])
-
-    serialize_status_attributes(object, status)
-
-    append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(status))
-    append_element(object, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(status.thread), href: ::TagManager.instance.url_for(status.thread)) unless status.thread.nil?
-    append_element(object, 'ostatus:conversation', nil, ref: conversation_uri(status.conversation)) unless status.conversation_id.nil?
-
-    object
-  end
-
-  def follow_salmon(follow)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{follow.account.acct} started following #{follow.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(follow.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:follow])
-
-    object = author(follow.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}")
-
-    entry << author(follow_request.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    object = author(follow_request.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def authorize_follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}")
-
-    entry << author(follow_request.target_account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:authorize])
-
-    object = Ox::Element.new('activity:object')
-    object << author(follow_request.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    inner_object = author(follow_request.target_account)
-    inner_object.value = 'activity:object'
-
-    object << inner_object
-    entry  << object
-    entry
-  end
-
-  def reject_follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}")
-
-    entry << author(follow_request.target_account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:reject])
-
-    object = Ox::Element.new('activity:object')
-    object << author(follow_request.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    inner_object = author(follow_request.target_account)
-    inner_object.value = 'activity:object'
-
-    object << inner_object
-    entry  << object
-    entry
-  end
-
-  def unfollow_salmon(follow)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(follow.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfollow])
-
-    object = author(follow.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def block_salmon(block)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
-    append_element(entry, 'title', description)
-
-    entry << author(block.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:block])
-
-    object = author(block.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def unblock_salmon(block)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{block.account.acct} no longer blocks #{block.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
-    append_element(entry, 'title', description)
-
-    entry << author(block.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unblock])
-
-    object = author(block.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def favourite_salmon(favourite)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(favourite.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:favorite])
-
-    entry << object(favourite.status)
-
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
-
-    entry
-  end
-
-  def unfavourite_salmon(favourite)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(favourite.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfavorite])
-
-    entry << object(favourite.status)
-
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
-
-    entry
-  end
-
-  private
-
-  def append_element(parent, name, content = nil, **attributes)
-    element = Ox::Element.new(name)
-    attributes.each { |k, v| element[k] = sanitize_str(v) }
-    element << sanitize_str(content) unless content.nil?
-    parent  << element
-  end
-
-  def sanitize_str(raw_str)
-    raw_str.to_s
-  end
-
-  def conversation_uri(conversation)
-    return conversation.uri if conversation.uri?
-    OStatus::TagManager.instance.unique_tag(conversation.created_at, conversation.id, 'Conversation')
-  end
-
-  def add_namespaces(parent)
-    parent['xmlns']          = OStatus::TagManager::XMLNS
-    parent['xmlns:thr']      = OStatus::TagManager::THR_XMLNS
-    parent['xmlns:activity'] = OStatus::TagManager::AS_XMLNS
-    parent['xmlns:poco']     = OStatus::TagManager::POCO_XMLNS
-    parent['xmlns:media']    = OStatus::TagManager::MEDIA_XMLNS
-    parent['xmlns:ostatus']  = OStatus::TagManager::OS_XMLNS
-    parent['xmlns:mastodon'] = OStatus::TagManager::MTDN_XMLNS
-  end
-
-  def serialize_status_attributes(entry, status)
-    append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(status)) if status.account.local?
-
-    append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
-    append_element(entry, 'content', Formatter.instance.format(status, inline_poll_options: true).to_str || '.', type: 'html', 'xml:lang': status.language)
-
-    status.active_mentions.sort_by(&:id).each do |mentioned|
-      append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))
-    end
-
-    append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:collection], href: OStatus::TagManager::COLLECTIONS[:public]) if status.public_visibility?
-
-    status.tags.each do |tag|
-      append_element(entry, 'category', nil, term: tag.name)
-    end
-
-    status.media_attachments.each do |media|
-      append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
-    end
-
-    append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? && status.media_attachments.any?
-    append_element(entry, 'mastodon:scope', status.visibility)
-
-    status.emojis.each do |emoji|
-      append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
-    end
-  end
-end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 5f7075a3c..9d874fe2c 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -40,8 +40,8 @@ class Request
     set_digest! if options.key?(:body)
   end
 
-  def on_behalf_of(account, key_id_format = :acct, sign_with: nil)
-    raise ArgumentError unless account.local?
+  def on_behalf_of(account, key_id_format = :uri, sign_with: nil)
+    raise ArgumentError, 'account must not be nil' if account.nil?
 
     @account       = account
     @keypair       = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
@@ -59,7 +59,7 @@ class Request
     begin
       response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
     rescue => e
-      raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
+      raise e.class, "#{e.message} on #{@url}", e.backtrace
     end
 
     begin
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
new file mode 100644
index 000000000..0cf1b8790
--- /dev/null
+++ b/app/lib/spam_check.rb
@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+class SpamCheck
+  include Redisable
+  include ActionView::Helpers::TextHelper
+
+  NILSIMSA_COMPARE_THRESHOLD = 95
+  NILSIMSA_MIN_SIZE          = 10
+  EXPIRE_SET_AFTER           = 1.week.seconds
+
+  def initialize(status)
+    @account = status.account
+    @status  = status
+  end
+
+  def skip?
+    disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
+  end
+
+  def spam?
+    if insufficient_data?
+      false
+    elsif nilsimsa?
+      any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
+    else
+      any_other_digest?('md5') { |_, other_digest| other_digest == digest }
+    end
+  end
+
+  def flag!
+    auto_silence_account!
+    auto_report_status!
+  end
+
+  def remember!
+    # The scores in sorted sets don't actually have enough bits to hold an exact
+    # value of our snowflake IDs, so we use it only for its ordering property. To
+    # get the correct status ID back, we have to save it in the string value
+
+    redis.zadd(redis_key, @status.id, digest_with_algorithm)
+    redis.zremrangebyrank(redis_key, '0', '-10')
+    redis.expire(redis_key, EXPIRE_SET_AFTER)
+  end
+
+  def reset!
+    redis.del(redis_key)
+  end
+
+  def hashable_text
+    return @hashable_text if defined?(@hashable_text)
+
+    @hashable_text = @status.text
+    @hashable_text = remove_mentions(@hashable_text)
+    @hashable_text = strip_tags(@hashable_text) unless @status.local?
+    @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
+    @hashable_text = remove_whitespace(@hashable_text)
+  end
+
+  def insufficient_data?
+    hashable_text.blank?
+  end
+
+  def digest
+    @digest ||= begin
+      if nilsimsa?
+        Nilsimsa.new(hashable_text).hexdigest
+      else
+        Digest::MD5.hexdigest(hashable_text)
+      end
+    end
+  end
+
+  def digest_with_algorithm
+    if nilsimsa?
+      ['nilsimsa', digest, @status.id].join(':')
+    else
+      ['md5', digest, @status.id].join(':')
+    end
+  end
+
+  private
+
+  def disabled?
+    !Setting.spam_check_enabled
+  end
+
+  def remove_mentions(text)
+    return text.gsub(Account::MENTION_RE, '') if @status.local?
+
+    Nokogiri::HTML.fragment(text).tap do |html|
+      mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
+
+      html.traverse do |element|
+        element.unlink if element.name == 'a' && mentions.include?(element['href'])
+      end
+    end.to_s
+  end
+
+  def normalize_unicode(text)
+    text.unicode_normalize(:nfkc).downcase
+  end
+
+  def remove_whitespace(text)
+    text.gsub(/\s+/, ' ').strip
+  end
+
+  def auto_silence_account!
+    @account.silence!
+  end
+
+  def auto_report_status!
+    status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
+    ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
+  end
+
+  def already_flagged?
+    @account.silenced?
+  end
+
+  def trusted?
+    @account.trust_level > Account::TRUST_LEVELS[:untrusted]
+  end
+
+  def no_unsolicited_mentions?
+    @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
+  end
+
+  def solicited_reply?
+    !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
+  end
+
+  def nilsimsa_compare_value(first, second)
+    first  = [first].pack('H*')
+    second = [second].pack('H*')
+    bits   = 0
+
+    0.upto(31) do |i|
+      bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
+    end
+
+    128 - bits # -128 <= Nilsimsa Compare Value <= 128
+  end
+
+  def nilsimsa?
+    hashable_text.size > NILSIMSA_MIN_SIZE
+  end
+
+  def other_digests
+    redis.zrange(redis_key, 0, -1)
+  end
+
+  def any_other_digest?(filter_algorithm)
+    other_digests.any? do |record|
+      algorithm, other_digest, status_id = record.split(':')
+
+      next unless algorithm == filter_algorithm
+
+      yield algorithm, other_digest, status_id
+    end
+  end
+
+  def matching_status_ids
+    if nilsimsa?
+      other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
+    else
+      other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
+    end
+  end
+
+  def redis_key
+    @redis_key ||= "spam_check:#{@account.id}"
+  end
+end
diff --git a/app/lib/status_finder.rb b/app/lib/status_finder.rb
index 4d1aed297..22ced8bf8 100644
--- a/app/lib/status_finder.rb
+++ b/app/lib/status_finder.rb
@@ -13,8 +13,6 @@ class StatusFinder
     raise ActiveRecord::RecordNotFound unless TagManager.instance.local_url?(url)
 
     case recognized_params[:controller]
-    when 'stream_entries'
-      StreamEntry.find(recognized_params[:id]).status
     when 'statuses'
       Status.find(recognized_params[:id])
     else
diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index fb364cb98..c88cf4994 100644
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -24,24 +24,16 @@ class TagManager
 
   def same_acct?(canonical, needle)
     return true if canonical.casecmp(needle).zero?
+
     username, domain = needle.split('@')
+
     local_domain?(domain) && canonical.casecmp(username).zero?
   end
 
   def local_url?(url)
     uri    = Addressable::URI.parse(url).normalize
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
-    TagManager.instance.web_domain?(domain)
-  end
-
-  def url_for(target)
-    return target.url if target.respond_to?(:local?) && !target.local?
 
-    case target.object_type
-    when :person
-      short_account_url(target)
-    when :note, :comment, :activity
-      short_account_status_url(target.account, target)
-    end
+    TagManager.instance.web_domain?(domain)
   end
 end
diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb
index a54a702a2..22d78874a 100644
--- a/app/lib/webfinger_resource.rb
+++ b/app/lib/webfinger_resource.rb
@@ -23,11 +23,17 @@ class WebfingerResource
   def username_from_url
     if account_show_page?
       path_params[:username]
+    elsif instance_actor_page?
+      Rails.configuration.x.local_domain
     else
       raise ActiveRecord::RecordNotFound
     end
   end
 
+  def instance_actor_page?
+    path_params[:controller] == 'instance_actors'
+  end
+
   def account_show_page?
     path_params[:controller] == 'accounts' && path_params[:action] == 'show'
   end