about summary refs log tree commit diff
path: root/lib
diff options
context:
space:
mode:
authorThibaut Girka <thib@sitedethib.com>2019-08-05 13:13:28 +0200
committerThibaut Girka <thib@sitedethib.com>2019-08-05 13:13:28 +0200
commite50554391aa726afe18ae1c0bc7ccbf69d7b4aec (patch)
treeb0cbd7b46a733f386c458a6f2f818a899f418b21 /lib
parentff0ceb28b3f1b19a6851a482f8203e434e50f167 (diff)
parent6201bfdfba7626c2b6bc5154dda1f41ee8c3ae71 (diff)
Merge branch 'master' into glitch-soc/merge-upstream
Diffstat (limited to 'lib')
-rw-r--r--lib/mastodon/domains_cli.rb18
1 files changed, 13 insertions, 5 deletions
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index f30062363..17cafd1bc 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -58,6 +58,7 @@ module Mastodon
     option :concurrency, type: :numeric, default: 50, aliases: [:c]
     option :silent, type: :boolean, default: false, aliases: [:s]
     option :format, type: :string, default: 'summary', aliases: [:f]
+    option :exclude_suspended, type: :boolean, default: false, aliases: [:x]
     desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START'
     long_desc <<-LONG_DESC
       Crawl the fediverse by using the Mastodon REST API endpoints that expose
@@ -74,18 +75,25 @@ module Mastodon
       default (`summary`), a summary of the statistics is returned. The other options
       are `domains`, which returns a newline-delimited list of all discovered peers,
       and `json`, which dumps all the aggregated data raw.
+
+      The --exclude-suspended (-x) option means that domains that are suspended
+      instance-wide do not appear in the output and are not included in summaries.
+      This also excludes subdomains of any of those domains.
     LONG_DESC
     def crawl(start = nil)
-      stats     = Concurrent::Hash.new
-      processed = Concurrent::AtomicFixnum.new(0)
-      failed    = Concurrent::AtomicFixnum.new(0)
-      start_at  = Time.now.to_f
-      seed      = start ? [start] : Account.remote.domains
+      stats           = Concurrent::Hash.new
+      processed       = Concurrent::AtomicFixnum.new(0)
+      failed          = Concurrent::AtomicFixnum.new(0)
+      start_at        = Time.now.to_f
+      seed            = start ? [start] : Account.remote.domains
+      blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$')
 
       pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
 
       work_unit = ->(domain) do
         next if stats.key?(domain)
+        next if options[:exclude_suspended] && domain.match(blocked_domains)
+
         stats[domain] = nil
         processed.increment