port tootsuite#11138 to monsterfork: Change domain blocks to automatically support subdomains

* Change domain blocks to automatically support subdomains

If a more authoritative domain is blocked (example.com), then the
same block will be applied to a subdomain (foo.example.com)

* Match subdomains of existing accounts when blocking/unblocking domains

* Improve code style

13 files changed, 76 insertions, 25 deletions

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index ec368470f..039742ac9 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -15,7 +15,7 @@ module Admin
       resource_params[:domain].strip! if resource_params[:domain].present?
       resource_params[:reason].strip! if resource_params[:reason].present?
       @domain_block = DomainBlock.new(resource_params)
-      existing_domain_block = resource_params[:domain].present? ? DomainBlock.find_by(domain: resource_params[:domain].strip) : nil
+      existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
 
       if existing_domain_block.present?
         @domain_block = existing_domain_block
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 28e14921f..155d515ea 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -21,7 +21,7 @@ module Admin
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.find_by(domain: params[:id])
+      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index d8da6ec22..e6d29992e 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -40,4 +40,8 @@ class MediaProxyController < ApplicationController
   def lock_options
     { redis: Redis.current, key: "media_download:#{params[:id]}" }
   end
+
+  def reject_media?
+    DomainBlock.reject_media?(@media_attachment.account.domain)
+  end
 end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 83330cb93..7b2d40ace 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -486,10 +486,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def skip_download?(remote_url = nil)
     return @skip_download if defined?(@skip_download)
-    domains = Set[@account.domain]
-    domains.add(remote_url.scan(/[\w\-]+\.[\w\-]+(?:\.[\w\-]+)*/).first) if remote_url.present?
-    blocks = DomainBlock.suspend.or(DomainBlock.where(reject_media: true))
-    @skip_download ||= domains.any? { |domain| blocks.where(domain: domain).or(blocks.where('domain LIKE ?', "%.#{domain}")).exists? }
+    @skip_download ||= DomainBlock.reject_media?(@account.domain)
   end
 
   def reply_to_local?
diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb
index 7fb6e3422..1423cedef 100644
--- a/app/lib/activitypub/activity/flag.rb
+++ b/app/lib/activitypub/activity/flag.rb
@@ -24,7 +24,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
   private
 
   def skip_reports?
-    DomainBlock.find_by(domain: @account.domain)&.reject_reports?
+    DomainBlock.reject_reports?(@account.domain)
   end
 
   def object_uris
diff --git a/app/models/account.rb b/app/models/account.rb
index 28ba35148..f50be9d4c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -122,6 +122,7 @@ class Account < ApplicationRecord
   scope :popular, -> { order('account_stats.followers_count desc') }
   scope :without_hidden, -> { where(hidden: false) }
   scope :without_unlisted, -> { where(unlisted: false) }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   delegate :email,
            :unconfirmed_email,
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 1aa1b5e34..33f59bc3d 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -39,6 +39,7 @@ class CustomEmoji < ApplicationRecord
   scope :local,      -> { where(domain: nil) }
   scope :remote,     -> { where.not(domain: nil) }
   scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   remotable_attachment :image, LIMIT
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 1d0b25772..05264190c 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -33,12 +33,38 @@ class DomainBlock < ApplicationRecord
 
   before_save :set_processing
 
-  def self.blocked?(domain)
-    suspend.where(domain: domain).or(suspend.where('domain LIKE ?', "%.#{domain}")).exists?
-  end
+  class << self
+    def suspend?(domain)
+      !!rule_for(domain)&.suspend?
+    end
+
+    def silence?(domain)
+      !!rule_for(domain)&.silence?
+    end
+
+    def reject_media?(domain)
+      !!rule_for(domain)&.reject_media?
+    end
+
+    def reject_reports?(domain)
+      !!rule_for(domain)&.reject_reports?
+    end
+
+    def force_unlisted?(domain)
+      !!rule_for(domain)&.severity == 'force_unlisted'
+    end
+
+    alias blocked? suspend?
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri      = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+      segments = uri.normalized_host.split('.')
+      variants = segments.map.with_index { |_, i| segments[i..-1].join('.') }
 
-  def self.force_unlisted?(domain)
-    where(domain: domain, severity: :force_unlisted).exists?
+      where(domain: variants[0..-2]).order(Arel.sql('char_length(domain) desc')).first
+    end
   end
 
   def stricter_than?(other_block)
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 08005f042..96a2c4b0b 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -256,8 +256,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def domain_block
     return @domain_block if defined?(@domain_block)
-
-    @domain_block = DomainBlock.find_by(domain: @domain)
+    @domain_block = DomainBlock.rule_for(@domain)
   end
 
   def key_changed?
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 36634fdd5..8ec77ce82 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -123,7 +123,7 @@ class BlockDomainService < BaseService
   end
 
   def blocked_domain_accounts
-    Account.where(domain: blocked_domain).reorder(nil)
+    Account.by_domain_and_subdomains(blocked_domain)
   end
 
   def media_from_blocked_domain
@@ -131,7 +131,7 @@ class BlockDomainService < BaseService
   end
 
   def emojis_from_blocked_domains
-    CustomEmoji.where(domain: blocked_domain)
+    CustomEmoji.by_domain_and_subdomains(blocked_domain)
   end
 
   def unknown_accounts
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index c0356a4b3..37c88d113 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -102,6 +102,8 @@ class ResolveAccountService < BaseService
     end
 
     @account
+  rescue Oj::ParseError
+    nil
   end
 
   def webfinger_update_due?
@@ -109,7 +111,10 @@ class ResolveAccountService < BaseService
   end
 
   def activitypub_ready?
-    !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type)
+    !@webfinger.link('self').nil? &&
+      ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) &&
+      !actor_json.nil? &&
+      actor_json['inbox'].present?
   end
 
   def actor_url
diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb
index c9130d90e..d8a6a25d0 100644
--- a/app/services/unblock_domain_service.rb
+++ b/app/services/unblock_domain_service.rb
@@ -25,7 +25,8 @@ class UnblockDomainService < BaseService
   end
 
   def blocked_accounts
-    scope = Account.where(domain: domain_block.domain)
+    scope = Account.by_domain_and_subdomains(domain_block.domain)
+
     if domain_block.silence?
       scope.where(silenced_at: @domain_block.created_at)
     else
diff --git a/spec/models/domain_block_spec.rb b/spec/models/domain_block_spec.rb
index 0035fd0ff..d98c5e118 100644
--- a/spec/models/domain_block_spec.rb
+++ b/spec/models/domain_block_spec.rb
@@ -21,23 +21,40 @@ RSpec.describe DomainBlock, type: :model do
     end
   end
 
-  describe 'blocked?' do
+  describe '.blocked?' do
     it 'returns true if the domain is suspended' do
-      Fabricate(:domain_block, domain: 'domain', severity: :suspend)
-      expect(DomainBlock.blocked?('domain')).to eq true
+      Fabricate(:domain_block, domain: 'example.com', severity: :suspend)
+      expect(DomainBlock.blocked?('example.com')).to eq true
     end
 
     it 'returns false even if the domain is silenced' do
-      Fabricate(:domain_block, domain: 'domain', severity: :silence)
-      expect(DomainBlock.blocked?('domain')).to eq false
+      Fabricate(:domain_block, domain: 'example.com', severity: :silence)
+      expect(DomainBlock.blocked?('example.com')).to eq false
     end
 
     it 'returns false if the domain is not suspended nor silenced' do
-      expect(DomainBlock.blocked?('domain')).to eq false
+      expect(DomainBlock.blocked?('example.com')).to eq false
     end
   end
 
-  describe 'stricter_than?' do
+  describe '.rule_for' do
+    it 'returns rule matching a blocked domain' do
+      block = Fabricate(:domain_block, domain: 'example.com')
+      expect(DomainBlock.rule_for('example.com')).to eq block
+    end
+
+    it 'returns a rule matching a subdomain of a blocked domain' do
+      block = Fabricate(:domain_block, domain: 'example.com')
+      expect(DomainBlock.rule_for('sub.example.com')).to eq block
+    end
+
+    it 'returns a rule matching a blocked subdomain' do
+      block = Fabricate(:domain_block, domain: 'sub.example.com')
+      expect(DomainBlock.rule_for('sub.example.com')).to eq block
+    end
+  end
+
+  describe '#stricter_than?' do
     it 'returns true if the new block has suspend severity while the old has lower severity' do
       suspend = DomainBlock.new(domain: 'domain', severity: :suspend)
       silence = DomainBlock.new(domain: 'domain', severity: :silence)