From 944de8a6c92cec50d99d460e076fe9bab6fc83ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 14 Jan 2019 10:13:34 +0100 Subject: Bump dotenv-rails from 2.5.0 to 2.6.0 (#9803) Bumps [dotenv-rails](https://github.com/bkeepers/dotenv) from 2.5.0 to 2.6.0. - [Release notes](https://github.com/bkeepers/dotenv/releases) - [Changelog](https://github.com/bkeepers/dotenv/blob/master/Changelog.md) - [Commits](https://github.com/bkeepers/dotenv/commits) Signed-off-by: dependabot[bot] --- Gemfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index ebf6c4656..e62116aea 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -185,9 +185,9 @@ GEM unf (>= 0.0.5, < 1.0.0) doorkeeper (5.0.2) railties (>= 4.2) - dotenv (2.5.0) - dotenv-rails (2.5.0) - dotenv (= 2.5.0) + dotenv (2.6.0) + dotenv-rails (2.6.0) + dotenv (= 2.6.0) railties (>= 3.2, < 6.0) elasticsearch (6.0.2) elasticsearch-api (= 6.0.2) @@ -200,7 +200,7 @@ GEM multi_json encryptor (3.0.0) equatable (0.5.0) - erubi (1.7.1) + erubi (1.8.0) et-orbi (1.1.6) tzinfo excon (0.62.0) @@ -269,7 +269,7 @@ GEM httplog (1.2.0) rack (>= 1.0) rainbow (>= 2.0.0) - i18n (1.1.1) + i18n (1.5.2) concurrent-ruby (~> 1.0) i18n-tasks (0.9.28) activesupport (>= 4.0.2) @@ -467,7 +467,7 @@ GEM rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) rainbow (3.0.0) - rake (12.3.1) + rake (12.3.2) rb-fsevent (0.10.3) rb-inotify (0.9.10) ffi (>= 0.5.0, < 2) @@ -678,7 +678,7 @@ DEPENDENCIES devise-two-factor (~> 3.0) devise_pam_authenticatable2 (~> 9.2) doorkeeper (~> 5.0) - dotenv-rails (~> 2.5) + dotenv-rails (~> 2.6) fabrication (~> 2.20) faker (~> 1.9) fast_blank (~> 1.0) -- cgit From f074d2f690a102bf56fb53d2f2c2c5824d8fc604 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 14 Jan 2019 10:15:55 +0100 Subject: Bump nsa from 0.2.4 to 0.2.7 (#9800) Bumps [nsa](https://github.com/localshred/nsa) from 0.2.4 to 0.2.7. - [Release notes](https://github.com/localshred/nsa/releases) - [Commits](https://github.com/localshred/nsa/commits) Signed-off-by: dependabot[bot] --- Gemfile.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index e62116aea..bcc5e164c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -148,7 +148,7 @@ GEM cocaine (0.5.8) climate_control (>= 0.0.3, < 1.0) coderay (1.1.2) - concurrent-ruby (1.0.5) + concurrent-ruby (1.1.4) connection_pool (2.2.2) crack (0.4.3) safe_yaml (~> 1.0.0) @@ -358,11 +358,11 @@ GEM mini_portile2 (~> 2.4.0) nokogumbo (2.0.0) nokogiri (~> 1.8, >= 1.8.4) - nsa (0.2.4) + nsa (0.2.7) activesupport (>= 4.2, < 6) - concurrent-ruby (~> 1.0.0) - sidekiq (>= 3.5.0) - statsd-ruby (~> 1.2.0) + concurrent-ruby (~> 1.0, >= 1.0.2) + sidekiq (>= 3.5) + statsd-ruby (~> 1.4, >= 1.4.0) oj (3.7.6) omniauth (1.9.0) hashie (>= 3.4.6, < 3.7.0) @@ -587,7 +587,7 @@ GEM net-scp (>= 1.1.2) net-ssh (>= 2.8.0) stackprof (0.2.12) - statsd-ruby (1.2.1) + statsd-ruby (1.4.0) stoplight (2.1.3) streamio-ffmpeg (3.0.2) multi_json (~> 1.8) -- cgit From 12307c564809604d503716da8e0f605f58a7f2c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 14 Jan 2019 10:16:43 +0100 Subject: Bump nokogiri from 1.10.0 to 1.10.1 (#9802) Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.10.0 to 1.10.1. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/master/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.10.0...v1.10.1) Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index bcc5e164c..f728d0850 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -354,7 +354,7 @@ GEM net-ssh (>= 2.6.5) net-ssh (5.0.2) nio4r (2.3.1) - nokogiri (1.10.0) + nokogiri (1.10.1) mini_portile2 (~> 2.4.0) nokogumbo (2.0.0) nokogiri (~> 1.8, >= 1.8.4) -- cgit From 5ac425ad28b6df9ec862851a9b326aaa4ab5c28c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 14 Jan 2019 10:18:55 +0100 Subject: Bump sidekiq-unique-jobs from 5.0.10 to 6.0.8 (#9801) Bumps [sidekiq-unique-jobs](https://github.com/mhenrixon/sidekiq-unique-jobs) from 5.0.10 to 6.0.8. - [Release notes](https://github.com/mhenrixon/sidekiq-unique-jobs/releases) - [Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/CHANGELOG.md) - [Commits](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v5.0.10...v6.0.8) Signed-off-by: dependabot[bot] --- Gemfile | 2 +- Gemfile.lock | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile b/Gemfile index 2204426ee..edec721d6 100644 --- a/Gemfile +++ b/Gemfile @@ -75,7 +75,7 @@ gem 'rqrcode', '~> 0.10' gem 'sanitize', '~> 5.0' gem 'sidekiq', '~> 5.2' gem 'sidekiq-scheduler', '~> 3.0' -gem 'sidekiq-unique-jobs', '~> 5.0' +gem 'sidekiq-unique-jobs', '~> 6.0' gem 'sidekiq-bulk', '~>0.2.0' gem 'simple-navigation', '~> 4.0' gem 'simple_form', '~> 4.1' diff --git a/Gemfile.lock b/Gemfile.lock index f728d0850..634054d83 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -563,8 +563,9 @@ GEM rufus-scheduler (~> 3.2) sidekiq (>= 3) tilt (>= 1.4.0) - sidekiq-unique-jobs (5.0.10) - sidekiq (>= 4.0, <= 6.0) + sidekiq-unique-jobs (6.0.8) + concurrent-ruby (~> 1.0, >= 1.0.5) + sidekiq (>= 4.0, < 6.0) thor (~> 0) simple-navigation (4.0.5) activesupport (>= 2.3.2) @@ -750,7 +751,7 @@ DEPENDENCIES sidekiq (~> 5.2) sidekiq-bulk (~> 0.2.0) sidekiq-scheduler (~> 3.0) - sidekiq-unique-jobs (~> 5.0) + sidekiq-unique-jobs (~> 6.0) simple-navigation (~> 4.0) simple_form (~> 4.1) simplecov (~> 0.16) -- cgit From a04b2bc728392de8845eee90ac5cca2567946190 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Tue, 15 Jan 2019 00:34:27 +0900 Subject: Bump pg from 1.1.3 to 1.1.4 (#9798) Bumps [pg](https://bitbucket.org/ged/ruby-pg) from 1.1.3 to 1.1.4. - [Changelog](https://bitbucket.org/ged/ruby-pg/src/default/History.rdoc) - [Commits](https://bitbucket.org/ged/ruby-pg/branches/compare/v1.1.4..v1.1.3) Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index 634054d83..a2019aa90 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -397,7 +397,7 @@ GEM pastel (0.7.2) equatable (~> 0.5.0) tty-color (~> 0.4.0) - pg (1.1.3) + pg (1.1.4) pghero (2.2.0) activerecord pkg-config (1.3.2) -- cgit From b4e6384aeafc9011707aa27d4948aaa9ca907db3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Tue, 15 Jan 2019 00:34:43 +0900 Subject: Bump aws-sdk-s3 from 1.30.0 to 1.30.1 (#9799) Bumps [aws-sdk-s3](https://github.com/aws/aws-sdk-ruby) from 1.30.0 to 1.30.1. - [Release notes](https://github.com/aws/aws-sdk-ruby/releases) - [Changelog](https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-ruby/commits) Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index a2019aa90..677281603 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,8 +76,8 @@ GEM av (0.9.0) cocaine (~> 0.5.3) aws-eventstream (1.0.1) - aws-partitions (1.122.0) - aws-sdk-core (3.43.0) + aws-partitions (1.131.0) + aws-sdk-core (3.45.0) aws-eventstream (~> 1.0) aws-partitions (~> 1.0) aws-sigv4 (~> 1.0) @@ -85,7 +85,7 @@ GEM aws-sdk-kms (1.13.0) aws-sdk-core (~> 3, >= 3.39.0) aws-sigv4 (~> 1.0) - aws-sdk-s3 (1.30.0) + aws-sdk-s3 (1.30.1) aws-sdk-core (~> 3, >= 3.39.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.0) -- cgit From ee5e24807fb59c398e7dfca807c51242269c7588 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 15 Jan 2019 09:24:35 +0100 Subject: Add `tootctl domains crawl` (#9809) --- Gemfile | 2 + Gemfile.lock | 5 +- lib/mastodon/domains_cli.rb | 118 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 2 deletions(-) (limited to 'Gemfile.lock') diff --git a/Gemfile b/Gemfile index edec721d6..fdfac330f 100644 --- a/Gemfile +++ b/Gemfile @@ -145,3 +145,5 @@ group :production do gem 'lograge', '~> 0.10' gem 'redis-rails', '~> 5.0' end + +gem 'concurrent-ruby', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 677281603..f96d24556 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -231,7 +231,7 @@ GEM fuubar (2.3.2) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) - get_process_mem (0.2.2) + get_process_mem (0.2.3) globalid (0.4.1) activesupport (>= 4.2.0) goldfinger (2.1.0) @@ -674,6 +674,7 @@ DEPENDENCIES chewy (~> 5.0) cld3 (~> 3.2.3) climate_control (~> 0.2) + concurrent-ruby derailed_benchmarks devise (~> 4.5) devise-two-factor (~> 3.0) @@ -773,4 +774,4 @@ RUBY VERSION ruby 2.5.3p105 BUNDLED WITH - 1.16.6 + 1.17.3 diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 16e298584..be68ae84b 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'concurrent' require_relative '../../config/boot' require_relative '../../config/environment' require_relative 'cli_helper' @@ -32,5 +33,122 @@ module Mastodon say say("Removed #{removed} accounts#{dry_run}", :green) end + + option :concurrency, type: :numeric, default: 50, aliases: [:c] + option :silent, type: :boolean, default: false, aliases: [:s] + option :format, type: :string, default: 'summary', aliases: [:f] + desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START' + long_desc <<-LONG_DESC + Crawl the fediverse by using the Mastodon REST API endpoints that expose + all known peers, and collect statistics from those peers, as long as those + peers support those API endpoints. When no START is given, the command uses + this server's own database of known peers to seed the crawl. + + The --concurrency (-c) option controls the number of threads performing HTTP + requests at the same time. More threads means the crawl may complete faster. + + The --silent (-s) option controls progress output. + + The --format (-f) option controls how the data is displayed at the end. By + default (`summary`), a summary of the statistics is returned. The other options + are `domains`, which returns a newline-delimited list of all discovered peers, + and `json`, which dumps all the aggregated data raw. + LONG_DESC + def crawl(start = nil) + stats = Concurrent::Hash.new + processed = Concurrent::AtomicFixnum.new(0) + failed = Concurrent::AtomicFixnum.new(0) + start_at = Time.now.to_f + seed = start ? [start] : Account.remote.domains + + pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0) + + work_unit = ->(domain) do + next if stats.key?(domain) + stats[domain] = nil + processed.increment + + begin + Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res| + next unless res.code == 200 + stats[domain] = Oj.load(res.to_s) + end + + Request.new(:get, "https://#{domain}/api/v1/instance/peers").perform do |res| + next unless res.code == 200 + + Oj.load(res.to_s).reject { |peer| stats.key?(peer) }.each do |peer| + pool.post(peer, &work_unit) + end + end + + Request.new(:get, "https://#{domain}/api/v1/instance/activity").perform do |res| + next unless res.code == 200 + stats[domain]['activity'] = Oj.load(res.to_s) + end + + say('.', :green, false) unless options[:silent] + rescue StandardError + failed.increment + say('.', :red, false) unless options[:silent] + end + end + + seed.each do |domain| + pool.post(domain, &work_unit) + end + + sleep 20 + sleep 20 until pool.queue_length.zero? + + pool.shutdown + pool.wait_for_termination(20) + ensure + pool.shutdown + + say unless options[:silent] + + case options[:format] + when 'summary' + stats_to_summary(stats, processed, failed, start_at) + when 'domains' + stats_to_domains(stats) + when 'json' + stats_to_json(stats) + end + end + + private + + def stats_to_summary(stats, processed, failed, start_at) + stats.compact! + + total_domains = stats.size + total_users = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['stats'].is_a?(Hash) ? sum + val['stats']['user_count'].to_i : sum } + total_active = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['logins'].to_i : sum } + total_joined = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['registrations'].to_i : sum } + + say("Visited #{processed.value} domains, #{failed.value} failed (#{(Time.now.to_f - start_at).round}s elapsed)", :green) + say("Total servers: #{total_domains}", :green) + say("Total registered: #{total_users}", :green) + say("Total active last week: #{total_active}", :green) + say("Total joined last week: #{total_joined}", :green) + end + + def stats_to_domains(stats) + say(stats.keys.join("\n")) + end + + def stats_to_json(stats) + totals.each_key do |domain| + if totals[domain].is_a?(Hash) + totals[domain]['activity'] = stats[domain] + else + totals.delete(domain) + end + end + + say(Oj.dump(totals)) + end end end -- cgit From 5363c724fc33e441cd4be22ba0e757b1ac4fd401 Mon Sep 17 00:00:00 2001 From: Daigo 3 Dango Date: Tue, 15 Jan 2019 22:11:59 +0000 Subject: User ruby2.6.0 on Heroku (#9818) --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Gemfile.lock') diff --git a/Gemfile.lock b/Gemfile.lock index f96d24556..8df0ac2e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -771,7 +771,7 @@ DEPENDENCIES webpush RUBY VERSION - ruby 2.5.3p105 + ruby 2.6.0p0 BUNDLED WITH 1.17.3 -- cgit