From ca17bae904783dfb1f4899a533d28a79da0c6fe9 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Tue, 25 Jun 2019 22:56:32 +0200 Subject: Use a redis-cached feed for the DM timeline --- app/workers/feed_insert_worker.rb | 11 ++++++++++- app/workers/scheduler/feed_cleanup_scheduler.rb | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/feed_insert_worker.rb b/app/workers/feed_insert_worker.rb index 1ae3c877b..546f5c0c2 100644 --- a/app/workers/feed_insert_worker.rb +++ b/app/workers/feed_insert_worker.rb @@ -13,6 +13,8 @@ class FeedInsertWorker when :list @list = List.find(id) @follower = @list.account + when :direct + @account = Account.find(id) end check_and_insert @@ -29,7 +31,12 @@ class FeedInsertWorker def feed_filtered? # Note: Lists are a variation of home, so the filtering rules # of home apply to both - FeedManager.instance.filter?(:home, @status, @follower.id) + case @type + when :home, :list + FeedManager.instance.filter?(:home, @status, @follower.id) + when :direct + FeedManager.instance.filter?(:direct, @status, @account.id) + end end def perform_push @@ -38,6 +45,8 @@ class FeedInsertWorker FeedManager.instance.push_to_home(@follower, @status) when :list FeedManager.instance.push_to_list(@list, @status) + when :direct + FeedManager.instance.push_to_direct(@account, @status) end end end diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb index bf5e20757..4933f1753 100644 --- a/app/workers/scheduler/feed_cleanup_scheduler.rb +++ b/app/workers/scheduler/feed_cleanup_scheduler.rb @@ -9,6 +9,7 @@ class Scheduler::FeedCleanupScheduler def perform clean_home_feeds! clean_list_feeds! + clean_direct_feeds! end private @@ -21,6 +22,10 @@ class Scheduler::FeedCleanupScheduler clean_feeds!(inactive_list_ids, :list) end + def clean_direct_feeds! + clean_feeds!(inactive_account_ids, :direct) + end + def clean_feeds!(ids, type) reblogged_id_sets = {} -- cgit From 0d9ffe56fb59e0d1fce91265f44140d874c0bfba Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:34:38 +0200 Subject: Add request pool to improve delivery performance (#10353) * Add request pool to improve delivery performance Fix #7909 * Ensure connection is closed when exception interrupts execution * Remove Timeout#timeout from socket connection * Fix infinite retrial loop on HTTP::ConnectionError * Close sockets on failure, reduce idle time to 90 seconds * Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server * Use a shared pool size, 512 by default, to stay below open file limit * Add some tests * Add more tests * Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds * Use a shared pool that returns preferred connection but re-purposes other ones when needed * Fix wrong connection being returned on subsequent calls within the same thread * Reduce mutex calls on flushes from 2 to 1 and add test for reaping --- Gemfile | 1 + Gemfile.lock | 1 + app/lib/connection_pool/shared_connection_pool.rb | 63 ++++++++++++ app/lib/connection_pool/shared_timed_stack.rb | 95 +++++++++++++++++ app/lib/request.rb | 70 +++++++++---- app/lib/request_pool.rb | 114 +++++++++++++++++++++ app/workers/activitypub/delivery_worker.rb | 15 ++- .../connection_pool/shared_connection_pool_spec.rb | 28 +++++ .../lib/connection_pool/shared_timed_stack_spec.rb | 61 +++++++++++ spec/lib/request_pool_spec.rb | 63 ++++++++++++ 10 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 app/lib/connection_pool/shared_connection_pool.rb create mode 100644 app/lib/connection_pool/shared_timed_stack.rb create mode 100644 app/lib/request_pool.rb create mode 100644 spec/lib/connection_pool/shared_connection_pool_spec.rb create mode 100644 spec/lib/connection_pool/shared_timed_stack_spec.rb create mode 100644 spec/lib/request_pool_spec.rb (limited to 'app/workers') diff --git a/Gemfile b/Gemfile index e99a808d4..2fcb603c3 100644 --- a/Gemfile +++ b/Gemfile @@ -148,3 +148,4 @@ group :production do end gem 'concurrent-ruby', require: false +gem 'connection_pool', require: false diff --git a/Gemfile.lock b/Gemfile.lock index c8856ccc5..6043a2fa9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -666,6 +666,7 @@ DEPENDENCIES cld3 (~> 3.2.4) climate_control (~> 0.2) concurrent-ruby + connection_pool derailed_benchmarks devise (~> 4.6) devise-two-factor (~> 3.0) diff --git a/app/lib/connection_pool/shared_connection_pool.rb b/app/lib/connection_pool/shared_connection_pool.rb new file mode 100644 index 000000000..2865a4108 --- /dev/null +++ b/app/lib/connection_pool/shared_connection_pool.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'connection_pool' +require_relative './shared_timed_stack' + +class ConnectionPool::SharedConnectionPool < ConnectionPool + def initialize(options = {}, &block) + super(options, &block) + + @available = ConnectionPool::SharedTimedStack.new(@size, &block) + end + + delegate :size, :flush, to: :@available + + def with(preferred_tag, options = {}) + Thread.handle_interrupt(Exception => :never) do + conn = checkout(preferred_tag, options) + + begin + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end + ensure + checkin(preferred_tag) + end + end + end + + def checkout(preferred_tag, options = {}) + if ::Thread.current[key(preferred_tag)] + ::Thread.current[key_count(preferred_tag)] += 1 + ::Thread.current[key(preferred_tag)] + else + ::Thread.current[key_count(preferred_tag)] = 1 + ::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout) + end + end + + def checkin(preferred_tag) + if ::Thread.current[key(preferred_tag)] + if ::Thread.current[key_count(preferred_tag)] == 1 + @available.push(::Thread.current[key(preferred_tag)]) + ::Thread.current[key(preferred_tag)] = nil + else + ::Thread.current[key_count(preferred_tag)] -= 1 + end + else + raise ConnectionPool::Error, 'no connections are checked out' + end + + nil + end + + private + + def key(tag) + :"#{@key}-#{tag}" + end + + def key_count(tag) + :"#{@key_count}-#{tag}" + end +end diff --git a/app/lib/connection_pool/shared_timed_stack.rb b/app/lib/connection_pool/shared_timed_stack.rb new file mode 100644 index 000000000..14a5285c4 --- /dev/null +++ b/app/lib/connection_pool/shared_timed_stack.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +class ConnectionPool::SharedTimedStack + def initialize(max = 0, &block) + @create_block = block + @max = max + @created = 0 + @queue = [] + @tagged_queue = Hash.new { |hash, key| hash[key] = [] } + @mutex = Mutex.new + @resource = ConditionVariable.new + end + + def push(connection) + @mutex.synchronize do + store_connection(connection) + @resource.broadcast + end + end + + alias << push + + def pop(preferred_tag, timeout = 5.0) + deadline = current_time + timeout + + @mutex.synchronize do + loop do + return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty? + + connection = try_create(preferred_tag) + return connection if connection + + to_wait = deadline - current_time + raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0 + + @resource.wait(@mutex, to_wait) + end + end + end + + def empty? + size.zero? + end + + def size + @mutex.synchronize do + @queue.size + end + end + + def flush + @mutex.synchronize do + @queue.delete_if do |connection| + delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME) + + if delete + @tagged_queue[connection.site].delete(connection) + connection.close + @created -= 1 + end + + delete + end + end + end + + private + + def try_create(preferred_tag) + if @created == @max && !@queue.empty? + throw_away_connection = @queue.pop + @tagged_queue[throw_away_connection.site].delete(throw_away_connection) + @create_block.call(preferred_tag) + elsif @created != @max + connection = @create_block.call(preferred_tag) + @created += 1 + connection + end + end + + def fetch_preferred_connection(preferred_tag) + connection = @tagged_queue[preferred_tag].pop + @queue.delete(connection) + connection + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def store_connection(connection) + @tagged_queue[connection.site].push(connection) + @queue.push(connection) + end +end diff --git a/app/lib/request.rb b/app/lib/request.rb index e555ae6a1..af49d6c77 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -17,15 +17,21 @@ end class Request REQUEST_TARGET = '(request-target)' + # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening + # and 5s timeout on the TLS handshake, meaning the worst case should take + # about 15s in total + TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze + include RoutingHelper def initialize(verb, url, **options) raise ArgumentError if url.blank? - @verb = verb - @url = Addressable::URI.parse(url).normalize - @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) - @headers = {} + @verb = verb + @url = Addressable::URI.parse(url).normalize + @http_client = options.delete(:http_client) + @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) + @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -50,15 +56,24 @@ class Request def perform begin - response = http_client.headers(headers).public_send(@verb, @url.to_s, @options) + response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers)) rescue => e raise e.class, "#{e.message} on #{@url}", e.backtrace[0] end begin - yield response.extend(ClientLimit) if block_given? + response = response.extend(ClientLimit) + + # If we are using a persistent connection, we have to + # read every response to be able to move forward at all. + # However, simply calling #to_s or #flush may not be safe, + # as the response body, if malicious, could be too big + # for our memory. So we use the #body_with_limit method + response.body_with_limit if http_client.persistent? + + yield response if block_given? ensure - http_client.close + http_client.close unless http_client.persistent? end end @@ -76,6 +91,10 @@ class Request %w(http https).include?(parsed_url.scheme) && parsed_url.host.present? end + + def http_client + HTTP.use(:auto_inflate).timeout(:per_operation, TIMEOUT.dup).follow(max_hops: 2) + end end private @@ -116,16 +135,8 @@ class Request end end - def timeout - # We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening - # and 5s timeout on the TLS handshake, meaning the worst case should take - # about 16s in total - - { connect: 5, read: 10, write: 10 } - end - def http_client - @http_client ||= HTTP.use(:auto_inflate).timeout(:per_operation, timeout).follow(max_hops: 2) + @http_client ||= Request.http_client end def use_proxy? @@ -169,20 +180,41 @@ class Request return super(host, *args) if thru_hidden_service?(host) outer_e = nil + port = args.first Resolv::DNS.open do |dns| dns.timeouts = 5 addresses = dns.getaddresses(host).take(2) - time_slot = 10.0 / addresses.size addresses.each do |address| begin raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s)) - ::Timeout.timeout(time_slot, HTTP::TimeoutError) do - return super(address.to_s, *args) + sock = ::Socket.new(::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) + sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) + + sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1) + + begin + sock.connect_nonblock(sockaddr) + rescue IO::WaitWritable + if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect]) + begin + sock.connect_nonblock(sockaddr) + rescue Errno::EISCONN + # Yippee! + rescue + sock.close + raise + end + else + sock.close + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + end end + + return sock rescue => e outer_e = e end diff --git a/app/lib/request_pool.rb b/app/lib/request_pool.rb new file mode 100644 index 000000000..e5899a79a --- /dev/null +++ b/app/lib/request_pool.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative './connection_pool/shared_connection_pool' + +class RequestPool + def self.current + @current ||= RequestPool.new + end + + class Reaper + attr_reader :pool, :frequency + + def initialize(pool, frequency) + @pool = pool + @frequency = frequency + end + + def run + return unless frequency&.positive? + + Thread.new(frequency, pool) do |t, p| + loop do + sleep t + p.flush + end + end + end + end + + MAX_IDLE_TIME = 30 + WAIT_TIMEOUT = 5 + MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i + + class Connection + attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh + + def initialize(site) + @site = site + @http_client = http_client + @last_used_at = nil + @created_at = current_time + @dead = false + @fresh = true + end + + def use + @last_used_at = current_time + @in_use = true + + retries = 0 + + begin + yield @http_client + rescue HTTP::ConnectionError + # It's possible the connection was closed, so let's + # try re-opening it once + + close + + if @fresh || retries.positive? + raise + else + @http_client = http_client + retries += 1 + retry + end + rescue StandardError + # If this connection raises errors of any kind, it's + # better if it gets reaped as soon as possible + + close + @dead = true + raise + end + ensure + @fresh = false + @in_use = false + end + + def seconds_idle + current_time - (@last_used_at || @created_at) + end + + def close + @http_client.close + end + + private + + def http_client + Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME) + end + + def current_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + end + + def initialize + @pool = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) } + @reaper = Reaper.new(self, 30) + @reaper.run + end + + def with(site, &block) + @pool.with(site) do |connection| + ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do + connection.use(&block) + end + end + end + + delegate :size, :flush, to: :@pool +end diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 5e4c391f0..79f1e8153 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -17,6 +17,7 @@ class ActivityPub::DeliveryWorker @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url + @host = Addressable::URI.parse(inbox_url).normalized_site perform_request @@ -28,16 +29,18 @@ class ActivityPub::DeliveryWorker private - def build_request - request = Request.new(:post, @inbox_url, body: @json) + def build_request(http_client) + request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) request.add_headers(HEADERS) end def perform_request light = Stoplight(@inbox_url) do - build_request.perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + request_pool.with(@host) do |http_client| + build_request(http_client).perform do |response| + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + end end end @@ -57,4 +60,8 @@ class ActivityPub::DeliveryWorker def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end + + def request_pool + RequestPool.current + end end diff --git a/spec/lib/connection_pool/shared_connection_pool_spec.rb b/spec/lib/connection_pool/shared_connection_pool_spec.rb new file mode 100644 index 000000000..114464558 --- /dev/null +++ b/spec/lib/connection_pool/shared_connection_pool_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedConnectionPool do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } } + + describe '#with' do + it 'runs a block with a connection' do + block_run = false + + subject.with('foo') do |connection| + expect(connection).to be_a MiniConnection + block_run = true + end + + expect(block_run).to be true + end + end +end diff --git a/spec/lib/connection_pool/shared_timed_stack_spec.rb b/spec/lib/connection_pool/shared_timed_stack_spec.rb new file mode 100644 index 000000000..f680c5966 --- /dev/null +++ b/spec/lib/connection_pool/shared_timed_stack_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe ConnectionPool::SharedTimedStack do + class MiniConnection + attr_reader :site + + def initialize(site) + @site = site + end + end + + subject { described_class.new(5) { |site| MiniConnection.new(site) } } + + describe '#push' do + it 'keeps the connection in the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.size).to eq 1 + end + end + + describe '#pop' do + it 'returns a connection' do + expect(subject.pop('foo')).to be_a MiniConnection + end + + it 'returns the same connection that was pushed in' do + connection = MiniConnection.new('foo') + subject.push(connection) + expect(subject.pop('foo')).to be connection + end + + it 'does not create more than maximum amount of connections' do + expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error + end + + it 'repurposes a connection for a different site when maximum amount is reached' do + 5.times { subject.push(MiniConnection.new('foo')) } + expect(subject.pop('bar')).to be_a MiniConnection + end + end + + describe '#empty?' do + it 'returns true when no connections on the stack' do + expect(subject.empty?).to be true + end + + it 'returns false when there are connections on the stack' do + subject.push(MiniConnection.new('foo')) + expect(subject.empty?).to be false + end + end + + describe '#size' do + it 'returns the number of connections on the stack' do + 2.times { subject.push(MiniConnection.new('foo')) } + expect(subject.size).to eq 2 + end + end +end diff --git a/spec/lib/request_pool_spec.rb b/spec/lib/request_pool_spec.rb new file mode 100644 index 000000000..4a144d7c7 --- /dev/null +++ b/spec/lib/request_pool_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe RequestPool do + subject { described_class.new } + + describe '#with' do + it 'returns a HTTP client for a host' do + subject.with('http://example.com') do |http_client| + expect(http_client).to be_a HTTP::Client + end + end + + it 'returns the same instance of HTTP client within the same thread for the same host' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.com') { |http_client| expect(http_client).to be test_client } + end + + it 'returns different HTTP clients for different hosts' do + test_client = nil + + subject.with('http://example.com') { |http_client| test_client = http_client } + expect(test_client).to_not be_nil + subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client } + end + + it 'grows to the number of threads accessing it' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject + + threads = 20.times.map do |i| + Thread.new do + 20.times do + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + end + end + end + + threads.map(&:join) + + expect(subject.size).to be > 1 + end + + it 'closes idle connections' do + stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!') + + subject.with('http://example.com') do |http_client| + http_client.get('/').flush + end + + expect(subject.size).to eq 1 + sleep RequestPool::MAX_IDLE_TIME + 30 + 1 + expect(subject.size).to eq 0 + end + end +end -- cgit From bc60d794f8411daa3c4c1541fb373309e4c029b6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 2 Jul 2019 00:59:53 +0200 Subject: Change ActivityPub::DeliveryWorker to not retry HTTP 501 errors (#11233) --- app/workers/activitypub/delivery_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 79f1e8153..818fd8f5d 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -54,7 +54,7 @@ class ActivityPub::DeliveryWorker end def response_error_unsalvageable?(response) - (400...500).cover?(response.code) && ![401, 408, 429].include?(response.code) + response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) end def failure_tracker -- cgit From 23aeef52cc4540b4514e9f3b935b21f0530a3746 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 6 Jul 2019 23:26:16 +0200 Subject: Remove Salmon and PubSubHubbub (#11205) * Remove Salmon and PubSubHubbub endpoints * Add error when trying to follow OStatus accounts * Fix new accounts not being created in ResolveAccountService --- app/controllers/activitypub/inboxes_controller.rb | 1 - app/controllers/admin/accounts_controller.rb | 16 +- app/controllers/api/push_controller.rb | 73 ------ app/controllers/api/salmon_controller.rb | 37 --- app/controllers/api/subscriptions_controller.rb | 51 ----- app/controllers/api/v1/follows_controller.rb | 31 --- app/lib/ostatus/activity/base.rb | 71 ------ app/lib/ostatus/activity/creation.rb | 219 ------------------ app/lib/ostatus/activity/deletion.rb | 16 -- app/lib/ostatus/activity/general.rb | 20 -- app/lib/ostatus/activity/post.rb | 23 -- app/lib/ostatus/activity/remote.rb | 11 - app/lib/ostatus/activity/share.rb | 26 --- app/lib/ostatus/atom_serializer.rb | 2 - app/models/account.rb | 3 +- app/serializers/webfinger_serializer.rb | 1 - app/services/authorize_follow_service.rb | 12 +- app/services/batched_remove_status_service.rb | 34 +-- app/services/block_domain_service.rb | 1 - app/services/block_service.rb | 12 +- app/services/concerns/author_extractor.rb | 23 -- app/services/concerns/stream_entry_renderer.rb | 7 - app/services/favourite_service.rb | 6 - app/services/fetch_remote_account_service.rb | 28 --- app/services/fetch_remote_status_service.rb | 28 --- app/services/follow_service.rb | 24 +- app/services/post_status_service.rb | 1 - app/services/process_feed_service.rb | 31 --- app/services/process_interaction_service.rb | 151 ------------ app/services/process_mentions_service.rb | 7 - app/services/pubsubhubbub/subscribe_service.rb | 53 ----- app/services/pubsubhubbub/unsubscribe_service.rb | 31 --- app/services/reblog_service.rb | 4 - app/services/reject_follow_service.rb | 12 +- app/services/remove_status_service.rb | 13 -- app/services/resolve_account_service.rb | 131 +---------- app/services/send_interaction_service.rb | 39 ---- app/services/subscribe_service.rb | 58 ----- app/services/unblock_service.rb | 12 +- app/services/unfavourite_service.rb | 13 +- app/services/unfollow_service.rb | 16 +- app/services/unsubscribe_service.rb | 36 --- app/services/update_remote_profile_service.rb | 66 ------ app/services/verify_salmon_service.rb | 26 --- app/views/accounts/show.html.haml | 1 - .../admin/subscriptions/_subscription.html.haml | 18 -- app/views/admin/subscriptions/index.html.haml | 16 -- app/views/well_known/webfinger/show.xml.ruby | 5 - app/workers/after_remote_follow_request_worker.rb | 24 +- app/workers/after_remote_follow_worker.rb | 24 +- app/workers/notification_worker.rb | 4 +- app/workers/processing_worker.rb | 4 +- app/workers/pubsubhubbub/confirmation_worker.rb | 75 +----- app/workers/pubsubhubbub/delivery_worker.rb | 74 +----- app/workers/pubsubhubbub/distribution_worker.rb | 25 +- .../pubsubhubbub/raw_distribution_worker.rb | 15 +- app/workers/pubsubhubbub/subscribe_worker.rb | 27 +-- app/workers/pubsubhubbub/unsubscribe_worker.rb | 8 +- app/workers/remote_profile_update_worker.rb | 6 +- app/workers/salmon_worker.rb | 6 +- app/workers/scheduler/subscriptions_scheduler.rb | 10 +- config/locales/en.yml | 7 - config/navigation.rb | 1 - config/routes.rb | 14 -- config/sidekiq.yml | 3 - spec/controllers/admin/accounts_controller_spec.rb | 38 ---- .../admin/subscriptions_controller_spec.rb | 32 --- spec/controllers/api/push_controller_spec.rb | 59 ----- spec/controllers/api/salmon_controller_spec.rb | 65 ------ .../api/subscriptions_controller_spec.rb | 68 ------ spec/controllers/api/v1/follows_controller_spec.rb | 51 ----- spec/fixtures/requests/webfinger.txt | 2 +- spec/lib/ostatus/atom_serializer_spec.rb | 145 ------------ spec/services/authorize_follow_service_spec.rb | 7 - .../services/batched_remove_status_service_spec.rb | 13 -- spec/services/block_service_spec.rb | 7 - spec/services/favourite_service_spec.rb | 7 - spec/services/fetch_remote_account_service_spec.rb | 40 ---- spec/services/follow_service_spec.rb | 68 ------ spec/services/import_service_spec.rb | 30 ++- spec/services/post_status_service_spec.rb | 2 - spec/services/process_feed_service_spec.rb | 252 --------------------- spec/services/process_interaction_service_spec.rb | 151 ------------ spec/services/process_mentions_service_spec.rb | 4 - .../pubsubhubbub/subscribe_service_spec.rb | 71 ------ .../pubsubhubbub/unsubscribe_service_spec.rb | 46 ---- spec/services/reblog_service_spec.rb | 4 - spec/services/reject_follow_service_spec.rb | 7 - spec/services/remove_status_service_spec.rb | 13 -- spec/services/resolve_account_service_spec.rb | 88 +------ spec/services/send_interaction_service_spec.rb | 7 - spec/services/subscribe_service_spec.rb | 43 ---- spec/services/unblock_service_spec.rb | 7 - spec/services/unfollow_service_spec.rb | 7 - spec/services/unsubscribe_service_spec.rb | 37 --- .../services/update_remote_profile_service_spec.rb | 84 ------- .../after_remote_follow_request_worker_spec.rb | 59 ----- spec/workers/after_remote_follow_worker_spec.rb | 59 ----- .../pubsubhubbub/confirmation_worker_spec.rb | 88 ------- spec/workers/pubsubhubbub/delivery_worker_spec.rb | 68 ------ .../pubsubhubbub/distribution_worker_spec.rb | 46 ---- .../scheduler/subscriptions_scheduler_spec.rb | 19 -- 102 files changed, 69 insertions(+), 3568 deletions(-) delete mode 100644 app/controllers/api/push_controller.rb delete mode 100644 app/controllers/api/salmon_controller.rb delete mode 100644 app/controllers/api/subscriptions_controller.rb delete mode 100644 app/controllers/api/v1/follows_controller.rb delete mode 100644 app/lib/ostatus/activity/base.rb delete mode 100644 app/lib/ostatus/activity/creation.rb delete mode 100644 app/lib/ostatus/activity/deletion.rb delete mode 100644 app/lib/ostatus/activity/general.rb delete mode 100644 app/lib/ostatus/activity/post.rb delete mode 100644 app/lib/ostatus/activity/remote.rb delete mode 100644 app/lib/ostatus/activity/share.rb delete mode 100644 app/services/concerns/author_extractor.rb delete mode 100644 app/services/concerns/stream_entry_renderer.rb delete mode 100644 app/services/process_feed_service.rb delete mode 100644 app/services/process_interaction_service.rb delete mode 100644 app/services/pubsubhubbub/subscribe_service.rb delete mode 100644 app/services/pubsubhubbub/unsubscribe_service.rb delete mode 100644 app/services/send_interaction_service.rb delete mode 100644 app/services/subscribe_service.rb delete mode 100644 app/services/unsubscribe_service.rb delete mode 100644 app/services/update_remote_profile_service.rb delete mode 100644 app/services/verify_salmon_service.rb delete mode 100644 app/views/admin/subscriptions/_subscription.html.haml delete mode 100644 app/views/admin/subscriptions/index.html.haml delete mode 100644 spec/controllers/admin/subscriptions_controller_spec.rb delete mode 100644 spec/controllers/api/push_controller_spec.rb delete mode 100644 spec/controllers/api/salmon_controller_spec.rb delete mode 100644 spec/controllers/api/subscriptions_controller_spec.rb delete mode 100644 spec/controllers/api/v1/follows_controller_spec.rb delete mode 100644 spec/services/process_feed_service_spec.rb delete mode 100644 spec/services/process_interaction_service_spec.rb delete mode 100644 spec/services/pubsubhubbub/subscribe_service_spec.rb delete mode 100644 spec/services/pubsubhubbub/unsubscribe_service_spec.rb delete mode 100644 spec/services/send_interaction_service_spec.rb delete mode 100644 spec/services/subscribe_service_spec.rb delete mode 100644 spec/services/unsubscribe_service_spec.rb delete mode 100644 spec/services/update_remote_profile_service_spec.rb delete mode 100644 spec/workers/after_remote_follow_request_worker_spec.rb delete mode 100644 spec/workers/after_remote_follow_worker_spec.rb delete mode 100644 spec/workers/pubsubhubbub/confirmation_worker_spec.rb delete mode 100644 spec/workers/pubsubhubbub/delivery_worker_spec.rb delete mode 100644 spec/workers/pubsubhubbub/distribution_worker_spec.rb delete mode 100644 spec/workers/scheduler/subscriptions_scheduler_spec.rb (limited to 'app/workers') diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb index a0b7532c2..e2cd8eaed 100644 --- a/app/controllers/activitypub/inboxes_controller.rb +++ b/app/controllers/activitypub/inboxes_controller.rb @@ -44,7 +44,6 @@ class ActivityPub::InboxesController < Api::BaseController ResolveAccountWorker.perform_async(signed_request_account.acct) end - Pubsubhubbub::UnsubscribeWorker.perform_async(signed_request_account.id) if signed_request_account.subscribed? DeliveryFailureTracker.track_inverse_success!(signed_request_account) end diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb index 0c7760d77..2fa1dfe5f 100644 --- a/app/controllers/admin/accounts_controller.rb +++ b/app/controllers/admin/accounts_controller.rb @@ -2,8 +2,8 @@ module Admin class AccountsController < BaseController - before_action :set_account, only: [:show, :subscribe, :unsubscribe, :redownload, :remove_avatar, :remove_header, :enable, :unsilence, :unsuspend, :memorialize, :approve, :reject] - before_action :require_remote_account!, only: [:subscribe, :unsubscribe, :redownload] + before_action :set_account, only: [:show, :redownload, :remove_avatar, :remove_header, :enable, :unsilence, :unsuspend, :memorialize, :approve, :reject] + before_action :require_remote_account!, only: [:redownload] before_action :require_local_account!, only: [:enable, :memorialize, :approve, :reject] def index @@ -19,18 +19,6 @@ module Admin @warnings = @account.targeted_account_warnings.latest.custom end - def subscribe - authorize @account, :subscribe? - Pubsubhubbub::SubscribeWorker.perform_async(@account.id) - redirect_to admin_account_path(@account.id) - end - - def unsubscribe - authorize @account, :unsubscribe? - Pubsubhubbub::UnsubscribeWorker.perform_async(@account.id) - redirect_to admin_account_path(@account.id) - end - def memorialize authorize @account, :memorialize? @account.memorialize! diff --git a/app/controllers/api/push_controller.rb b/app/controllers/api/push_controller.rb deleted file mode 100644 index e04d19125..000000000 --- a/app/controllers/api/push_controller.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -class Api::PushController < Api::BaseController - include SignatureVerification - - def update - response, status = process_push_request - render plain: response, status: status - end - - private - - def process_push_request - case hub_mode - when 'subscribe' - Pubsubhubbub::SubscribeService.new.call(account_from_topic, hub_callback, hub_secret, hub_lease_seconds, verified_domain) - when 'unsubscribe' - Pubsubhubbub::UnsubscribeService.new.call(account_from_topic, hub_callback) - else - ["Unknown mode: #{hub_mode}", 422] - end - end - - def hub_mode - params['hub.mode'] - end - - def hub_topic - params['hub.topic'] - end - - def hub_callback - params['hub.callback'] - end - - def hub_lease_seconds - params['hub.lease_seconds'] - end - - def hub_secret - params['hub.secret'] - end - - def account_from_topic - if hub_topic.present? && local_domain? && account_feed_path? - Account.find_local(hub_topic_params[:username]) - end - end - - def hub_topic_params - @_hub_topic_params ||= Rails.application.routes.recognize_path(hub_topic_uri.path) - end - - def hub_topic_uri - @_hub_topic_uri ||= Addressable::URI.parse(hub_topic).normalize - end - - def local_domain? - TagManager.instance.web_domain?(hub_topic_domain) - end - - def verified_domain - return signed_request_account.domain if signed_request_account - end - - def hub_topic_domain - hub_topic_uri.host + (hub_topic_uri.port ? ":#{hub_topic_uri.port}" : '') - end - - def account_feed_path? - hub_topic_params[:controller] == 'accounts' && hub_topic_params[:action] == 'show' && hub_topic_params[:format] == 'atom' - end -end diff --git a/app/controllers/api/salmon_controller.rb b/app/controllers/api/salmon_controller.rb deleted file mode 100644 index ac5f3268d..000000000 --- a/app/controllers/api/salmon_controller.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -class Api::SalmonController < Api::BaseController - include SignatureVerification - - before_action :set_account - respond_to :txt - - def update - if verify_payload? - process_salmon - head 202 - elsif payload.present? - render plain: signature_verification_failure_reason, status: 401 - else - head 400 - end - end - - private - - def set_account - @account = Account.find(params[:id]) - end - - def payload - @_payload ||= request.body.read - end - - def verify_payload? - payload.present? && VerifySalmonService.new.call(payload) - end - - def process_salmon - SalmonWorker.perform_async(@account.id, payload.force_encoding('UTF-8')) - end -end diff --git a/app/controllers/api/subscriptions_controller.rb b/app/controllers/api/subscriptions_controller.rb deleted file mode 100644 index 89007f3d6..000000000 --- a/app/controllers/api/subscriptions_controller.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -class Api::SubscriptionsController < Api::BaseController - before_action :set_account - respond_to :txt - - def show - if subscription.valid?(params['hub.topic']) - @account.update(subscription_expires_at: future_expires) - render plain: encoded_challenge, status: 200 - else - head 404 - end - end - - def update - if subscription.verify(body, request.headers['HTTP_X_HUB_SIGNATURE']) - ProcessingWorker.perform_async(@account.id, body.force_encoding('UTF-8')) - end - - head 200 - end - - private - - def subscription - @_subscription ||= @account.subscription( - api_subscription_url(@account.id) - ) - end - - def body - @_body ||= request.body.read - end - - def encoded_challenge - HTMLEntities.new.encode(params['hub.challenge']) - end - - def future_expires - Time.now.utc + lease_seconds_or_default - end - - def lease_seconds_or_default - (params['hub.lease_seconds'] || 1.day).to_i.seconds - end - - def set_account - @account = Account.find(params[:id]) - end -end diff --git a/app/controllers/api/v1/follows_controller.rb b/app/controllers/api/v1/follows_controller.rb deleted file mode 100644 index 5420c0533..000000000 --- a/app/controllers/api/v1/follows_controller.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class Api::V1::FollowsController < Api::BaseController - before_action -> { doorkeeper_authorize! :follow, :'write:follows' } - before_action :require_user! - - respond_to :json - - def create - raise ActiveRecord::RecordNotFound if follow_params[:uri].blank? - - @account = FollowService.new.call(current_user.account, target_uri).try(:target_account) - - if @account.nil? - username, domain = target_uri.split('@') - @account = Account.find_remote!(username, domain) - end - - render json: @account, serializer: REST::AccountSerializer - end - - private - - def target_uri - follow_params[:uri].strip.gsub(/\A@/, '') - end - - def follow_params - params.permit(:uri) - end -end diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb deleted file mode 100644 index db70f1998..000000000 --- a/app/lib/ostatus/activity/base.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Base - include Redisable - - def initialize(xml, account = nil, **options) - @xml = xml - @account = account - @options = options - end - - def status? - [:activity, :note, :comment].include?(type) - end - - def verb - raw = @xml.at_xpath('./activity:verb', activity: OStatus::TagManager::AS_XMLNS).content - OStatus::TagManager::VERBS.key(raw) - rescue - :post - end - - def type - raw = @xml.at_xpath('./activity:object-type', activity: OStatus::TagManager::AS_XMLNS).content - OStatus::TagManager::TYPES.key(raw) - rescue - :activity - end - - def id - @xml.at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content - end - - def url - link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| link_candidate['type'] == 'text/html' } - link.nil? ? nil : link['href'] - end - - def activitypub_uri - link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link_candidate['type']) } - link.nil? ? nil : link['href'] - end - - def activitypub_uri? - activitypub_uri.present? - end - - private - - def find_status(uri) - if OStatus::TagManager.instance.local_id?(uri) - local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status') - return Status.find_by(id: local_id) - elsif ActivityPub::TagManager.instance.local_uri?(uri) - local_id = ActivityPub::TagManager.instance.uri_to_local_id(uri) - return Status.find_by(id: local_id) - end - - Status.find_by(uri: uri) - end - - def find_activitypub_status(uri, href) - tag_matches = /tag:([^,:]+)[^:]*:objectId=([\d]+)/.match(uri) - href_matches = %r{/users/([^/]+)}.match(href) - - unless tag_matches.nil? || href_matches.nil? - uri = "https://#{tag_matches[1]}/users/#{href_matches[1]}/statuses/#{tag_matches[2]}" - Status.find_by(uri: uri) - end - end -end diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb deleted file mode 100644 index 60de712db..000000000 --- a/app/lib/ostatus/activity/creation.rb +++ /dev/null @@ -1,219 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Creation < OStatus::Activity::Base - def perform - if redis.exists("delete_upon_arrival:#{@account.id}:#{id}") - Rails.logger.debug "Delete for status #{id} was queued, ignoring" - return [nil, false] - end - - return [nil, false] if @account.suspended? || invalid_origin? - - RedisLock.acquire(lock_options) do |lock| - if lock.acquired? - # Return early if status already exists in db - @status = find_status(id) - return [@status, false] unless @status.nil? - @status = process_status - else - raise Mastodon::RaceConditionError - end - end - - [@status, true] - end - - def process_status - Rails.logger.debug "Creating remote status #{id}" - cached_reblog = reblog - status = nil - - # Skip if the reblogged status is not public - return if cached_reblog && !(cached_reblog.public_visibility? || cached_reblog.unlisted_visibility?) - - media_attachments = save_media.take(4) - - ApplicationRecord.transaction do - status = Status.create!( - uri: id, - url: url, - account: @account, - reblog: cached_reblog, - text: content, - spoiler_text: content_warning, - created_at: published, - override_timestamps: @options[:override_timestamps], - reply: thread?, - language: content_language, - visibility: visibility_scope, - conversation: find_or_create_conversation, - thread: thread? ? find_status(thread.first) || find_activitypub_status(thread.first, thread.second) : nil, - media_attachment_ids: media_attachments.map(&:id), - sensitive: sensitive? - ) - - save_mentions(status) - save_hashtags(status) - save_emojis(status) - end - - if thread? && status.thread.nil? && Request.valid_url?(thread.second) - Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}" - ThreadResolveWorker.perform_async(status.id, thread.second) - end - - Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" - - LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? - - # Only continue if the status is supposed to have arrived in real-time. - # Note that if @options[:override_timestamps] isn't set, the status - # may have a lower snowflake id than other existing statuses, potentially - # "hiding" it from paginated API calls - return status unless @options[:override_timestamps] || status.within_realtime_window? - - DistributionWorker.perform_async(status.id) - - status - end - - def content - @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content - end - - def content_language - @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS)['xml:lang']&.presence || 'en' - end - - def content_warning - @xml.at_xpath('./xmlns:summary', xmlns: OStatus::TagManager::XMLNS)&.content || '' - end - - def visibility_scope - @xml.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content&.to_sym || :public - end - - def published - @xml.at_xpath('./xmlns:published', xmlns: OStatus::TagManager::XMLNS).content - end - - def thread? - !@xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS).nil? - end - - def thread - thr = @xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS) - [thr['ref'], thr['href']] - end - - private - - def sensitive? - # OStatus-specific convention (not standard) - @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).any? { |category| category['term'] == 'nsfw' } - end - - def find_or_create_conversation - uri = @xml.at_xpath('./ostatus:conversation', ostatus: OStatus::TagManager::OS_XMLNS)&.attribute('ref')&.content - return if uri.nil? - - if OStatus::TagManager.instance.local_id?(uri) - local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation') - return Conversation.find_by(id: local_id) - end - - Conversation.find_by(uri: uri) || Conversation.create!(uri: uri) - end - - def save_mentions(parent) - processed_account_ids = [] - - @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each do |link| - next if [OStatus::TagManager::TYPES[:group], OStatus::TagManager::TYPES[:collection]].include? link['ostatus:object-type'] - - mentioned_account = account_from_href(link['href']) - - next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id) - - mentioned_account.mentions.where(status: parent).first_or_create(status: parent) - - # So we can skip duplicate mentions - processed_account_ids << mentioned_account.id - end - end - - def save_hashtags(parent) - tags = @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).map { |category| category['term'] }.select(&:present?) - ProcessHashtagsService.new.call(parent, tags) - end - - def save_media - do_not_download = DomainBlock.reject_media?(@account.domain) - media_attachments = [] - - @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link| - next unless link['href'] - - media = MediaAttachment.where(status: nil, remote_url: link['href']).first_or_initialize(account: @account, status: nil, remote_url: link['href']) - parsed_url = Addressable::URI.parse(link['href']).normalize - - next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? - - media.save - media_attachments << media - - next if do_not_download - - begin - media.file_remote_url = link['href'] - media.save! - rescue ActiveRecord::RecordInvalid - next - end - end - - media_attachments - end - - def save_emojis(parent) - do_not_download = DomainBlock.reject_media?(parent.account.domain) - - return if do_not_download - - @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS).each do |link| - next unless link['href'] && link['name'] - - shortcode = link['name'].delete(':') - emoji = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain) - - next unless emoji.nil? - - emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain) - emoji.image_remote_url = link['href'] - emoji.save - end - end - - def account_from_href(href) - url = Addressable::URI.parse(href).normalize - - if TagManager.instance.web_domain?(url.host) - Account.find_local(url.path.gsub('/users/', '')) - else - Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href) - end - end - - def invalid_origin? - return false unless id.start_with?('http') # Legacy IDs cannot be checked - - needle = Addressable::URI.parse(id).normalized_host - - !(needle.casecmp(@account.domain).zero? || - needle.casecmp(Addressable::URI.parse(@account.remote_url.presence || @account.uri).normalized_host).zero?) - end - - def lock_options - { redis: Redis.current, key: "create:#{id}" } - end -end diff --git a/app/lib/ostatus/activity/deletion.rb b/app/lib/ostatus/activity/deletion.rb deleted file mode 100644 index c98f5ee0a..000000000 --- a/app/lib/ostatus/activity/deletion.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Deletion < OStatus::Activity::Base - def perform - Rails.logger.debug "Deleting remote status #{id}" - - status = Status.find_by(uri: id, account: @account) - status ||= Status.find_by(uri: activitypub_uri, account: @account) if activitypub_uri? - - if status.nil? - redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id) - else - RemoveStatusService.new.call(status) - end - end -end diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb deleted file mode 100644 index 8a6aabc33..000000000 --- a/app/lib/ostatus/activity/general.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::General < OStatus::Activity::Base - def specialize - special_class&.new(@xml, @account, @options) - end - - private - - def special_class - case verb - when :post - OStatus::Activity::Post - when :share - OStatus::Activity::Share - when :delete - OStatus::Activity::Deletion - end - end -end diff --git a/app/lib/ostatus/activity/post.rb b/app/lib/ostatus/activity/post.rb deleted file mode 100644 index 755ed8656..000000000 --- a/app/lib/ostatus/activity/post.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Post < OStatus::Activity::Creation - def perform - status, just_created = super - - if just_created - status.mentions.includes(:account).each do |mention| - mentioned_account = mention.account - next unless mentioned_account.local? - NotifyService.new.call(mentioned_account, mention) - end - end - - status - end - - private - - def reblog - nil - end -end diff --git a/app/lib/ostatus/activity/remote.rb b/app/lib/ostatus/activity/remote.rb deleted file mode 100644 index 5b204b6d8..000000000 --- a/app/lib/ostatus/activity/remote.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Remote < OStatus::Activity::Base - def perform - if activitypub_uri? - find_status(activitypub_uri) || FetchRemoteStatusService.new.call(url) - else - find_status(id) || FetchRemoteStatusService.new.call(url) - end - end -end diff --git a/app/lib/ostatus/activity/share.rb b/app/lib/ostatus/activity/share.rb deleted file mode 100644 index 5ca601415..000000000 --- a/app/lib/ostatus/activity/share.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -class OStatus::Activity::Share < OStatus::Activity::Creation - def perform - return if reblog.nil? - - status, just_created = super - NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created - status - end - - def object - @xml.at_xpath('.//activity:object', activity: OStatus::TagManager::AS_XMLNS) - end - - private - - def reblog - return @reblog if defined? @reblog - - original_status = OStatus::Activity::Remote.new(object).perform - return if original_status.nil? - - @reblog = original_status.reblog? ? original_status.reblog : original_status - end -end diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb index 9a05d96cf..f5c0e85ca 100644 --- a/app/lib/ostatus/atom_serializer.rb +++ b/app/lib/ostatus/atom_serializer.rb @@ -53,8 +53,6 @@ class OStatus::AtomSerializer append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account)) append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom')) append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20 - append_element(feed, 'link', nil, rel: :hub, href: api_push_url) - append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id)) stream_entries.each do |stream_entry| feed << entry(stream_entry) diff --git a/app/models/account.rb b/app/models/account.rb index c588451fc..d6772eb98 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -164,8 +164,7 @@ class Account < ApplicationRecord end def refresh! - return if local? - ResolveAccountService.new.call(acct) + ResolveAccountService.new.call(acct) unless local? end def silenced? diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb index 8c0b07702..4220f697e 100644 --- a/app/serializers/webfinger_serializer.rb +++ b/app/serializers/webfinger_serializer.rb @@ -18,7 +18,6 @@ class WebfingerSerializer < ActiveModel::Serializer { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) }, { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') }, { rel: 'self', type: 'application/activity+json', href: account_url(object) }, - { rel: 'salmon', href: api_salmon_url(object.id) }, { rel: 'magic-public-key', href: "data:application/magic-public-key,#{object.magic_key}" }, { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" }, ] diff --git a/app/services/authorize_follow_service.rb b/app/services/authorize_follow_service.rb index 29b8700c7..49bef727e 100644 --- a/app/services/authorize_follow_service.rb +++ b/app/services/authorize_follow_service.rb @@ -11,25 +11,17 @@ class AuthorizeFollowService < BaseService follow_request.authorize! end - create_notification(follow_request) unless source_account.local? + create_notification(follow_request) if !source_account.local? && source_account.activitypub? follow_request end private def create_notification(follow_request) - if follow_request.account.ostatus? - NotificationWorker.perform_async(build_xml(follow_request), follow_request.target_account_id, follow_request.account_id) - elsif follow_request.account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url) - end + ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url) end def build_json(follow_request) Oj.dump(serialize_payload(follow_request, ActivityPub::AcceptFollowSerializer)) end - - def build_xml(follow_request) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)) - end end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index e328b1739..cb66debc8 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class BatchedRemoveStatusService < BaseService - include StreamEntryRenderer include Redisable # Delete given statuses and reblogs of them @@ -18,10 +17,7 @@ class BatchedRemoveStatusService < BaseService @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a } @tags = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) } - @stream_entry_batches = [] - @salmon_batches = [] - @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) } - @activity_xml = {} + @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) } # Ensure that rendered XML reflects destroyed state statuses.each do |status| @@ -39,28 +35,16 @@ class BatchedRemoveStatusService < BaseService unpush_from_home_timelines(account, account_statuses) unpush_from_list_timelines(account, account_statuses) - - batch_stream_entries(account, account_statuses) if account.local? end # Cannot be batched statuses.each do |status| unpush_from_public_timelines(status) - batch_salmon_slaps(status) if status.local? end - - Pubsubhubbub::RawDistributionWorker.push_bulk(@stream_entry_batches) { |batch| batch } - NotificationWorker.push_bulk(@salmon_batches) { |batch| batch } end private - def batch_stream_entries(account, statuses) - statuses.each do |status| - @stream_entry_batches << [build_xml(status.stream_entry), account.id] - end - end - def unpush_from_home_timelines(account, statuses) recipients = account.followers_for_local_distribution.to_a @@ -101,20 +85,4 @@ class BatchedRemoveStatusService < BaseService end end end - - def batch_salmon_slaps(status) - return if @mentions[status.id].empty? - - recipients = @mentions[status.id].map(&:account).reject(&:local?).select(&:ostatus?).uniq(&:domain).map(&:id) - - recipients.each do |recipient_id| - @salmon_batches << [build_xml(status.stream_entry), status.account_id, recipient_id] - end - end - - def build_xml(stream_entry) - return @activity_xml[stream_entry.id] if @activity_xml.key?(stream_entry.id) - - @activity_xml[stream_entry.id] = stream_entry_to_xml(stream_entry) - end end diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb index c6eef04d4..c5e5e5761 100644 --- a/app/services/block_domain_service.rb +++ b/app/services/block_domain_service.rb @@ -44,7 +44,6 @@ class BlockDomainService < BaseService def suspend_accounts! blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account| - UnsubscribeService.new.call(account) if account.subscribed? SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at) end end diff --git a/app/services/block_service.rb b/app/services/block_service.rb index 9050a4858..da06361c2 100644 --- a/app/services/block_service.rb +++ b/app/services/block_service.rb @@ -13,25 +13,17 @@ class BlockService < BaseService block = account.block!(target_account) BlockWorker.perform_async(account.id, target_account.id) - create_notification(block) unless target_account.local? + create_notification(block) if !target_account.local? && target_account.activitypub? block end private def create_notification(block) - if block.target_account.ostatus? - NotificationWorker.perform_async(build_xml(block), block.account_id, block.target_account_id) - elsif block.target_account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(block), block.account_id, block.target_account.inbox_url) - end + ActivityPub::DeliveryWorker.perform_async(build_json(block), block.account_id, block.target_account.inbox_url) end def build_json(block) Oj.dump(serialize_payload(block, ActivityPub::BlockSerializer)) end - - def build_xml(block) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.block_salmon(block)) - end end diff --git a/app/services/concerns/author_extractor.rb b/app/services/concerns/author_extractor.rb deleted file mode 100644 index c2419e9ec..000000000 --- a/app/services/concerns/author_extractor.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module AuthorExtractor - def author_from_xml(xml, update_profile = true) - return nil if xml.nil? - - # Try for acct - acct = xml.at_xpath('./xmlns:author/xmlns:email', xmlns: OStatus::TagManager::XMLNS)&.content - - # Try + - if acct.blank? - username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: OStatus::TagManager::XMLNS)&.content - uri = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: OStatus::TagManager::XMLNS)&.content - - return nil if username.blank? || uri.blank? - - domain = Addressable::URI.parse(uri).normalized_host - acct = "#{username}@#{domain}" - end - - ResolveAccountService.new.call(acct, update_profile: update_profile) - end -end diff --git a/app/services/concerns/stream_entry_renderer.rb b/app/services/concerns/stream_entry_renderer.rb deleted file mode 100644 index 9f6c8a082..000000000 --- a/app/services/concerns/stream_entry_renderer.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module StreamEntryRenderer - def stream_entry_to_xml(stream_entry) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.entry(stream_entry, true)) - end -end diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb index 128a24ad6..02b26458a 100644 --- a/app/services/favourite_service.rb +++ b/app/services/favourite_service.rb @@ -30,8 +30,6 @@ class FavouriteService < BaseService if status.account.local? NotifyService.new.call(status.account, favourite) - elsif status.account.ostatus? - NotificationWorker.perform_async(build_xml(favourite), favourite.account_id, status.account_id) elsif status.account.activitypub? ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url) end @@ -46,8 +44,4 @@ class FavouriteService < BaseService def build_json(favourite) Oj.dump(serialize_payload(favourite, ActivityPub::LikeSerializer)) end - - def build_xml(favourite) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.favourite_salmon(favourite)) - end end diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb index cfc560022..a7f95603d 100644 --- a/app/services/fetch_remote_account_service.rb +++ b/app/services/fetch_remote_account_service.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class FetchRemoteAccountService < BaseService - include AuthorExtractor - def call(url, prefetched_body = nil, protocol = :ostatus) if prefetched_body.nil? resource_url, resource_options, protocol = FetchAtomService.new.call(url) @@ -12,34 +10,8 @@ class FetchRemoteAccountService < BaseService end case protocol - when :ostatus - process_atom(resource_url, **resource_options) when :activitypub ActivityPub::FetchRemoteAccountService.new.call(resource_url, **resource_options) end end - - private - - def process_atom(url, prefetched_body:) - xml = Nokogiri::XML(prefetched_body) - xml.encoding = 'utf-8' - - account = author_from_xml(xml.at_xpath('/xmlns:feed', xmlns: OStatus::TagManager::XMLNS), false) - - UpdateRemoteProfileService.new.call(xml, account) if account.present? && trusted_domain?(url, account) - - account - rescue TypeError - Rails.logger.debug "Unparseable URL given: #{url}" - nil - rescue Nokogiri::XML::XPath::SyntaxError - Rails.logger.debug 'Invalid XML or missing namespace' - nil - end - - def trusted_domain?(url, account) - domain = Addressable::URI.parse(url).normalized_host - domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url.presence || account.uri).normalized_host).zero? - end end diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb index 9c3008035..aac39dfd5 100644 --- a/app/services/fetch_remote_status_service.rb +++ b/app/services/fetch_remote_status_service.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class FetchRemoteStatusService < BaseService - include AuthorExtractor - def call(url, prefetched_body = nil, protocol = :ostatus) if prefetched_body.nil? resource_url, resource_options, protocol = FetchAtomService.new.call(url) @@ -12,34 +10,8 @@ class FetchRemoteStatusService < BaseService end case protocol - when :ostatus - process_atom(resource_url, **resource_options) when :activitypub ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options) end end - - private - - def process_atom(url, prefetched_body:) - Rails.logger.debug "Processing Atom for remote status at #{url}" - - xml = Nokogiri::XML(prefetched_body) - xml.encoding = 'utf-8' - - account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS)) - domain = Addressable::URI.parse(url).normalized_host - - return nil unless !account.nil? && confirmed_domain?(domain, account) - - statuses = ProcessFeedService.new.call(prefetched_body, account) - statuses.first - rescue Nokogiri::XML::XPath::SyntaxError - Rails.logger.debug 'Invalid XML or missing namespace' - nil - end - - def confirmed_domain?(domain, account) - account.domain.nil? || domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url.presence || account.uri).normalized_host).zero? - end end diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index 0305e2d62..8e118f5d3 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -13,7 +13,7 @@ class FollowService < BaseService target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true) raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended? - raise Mastodon::NotPermittedError if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? + raise Mastodon::NotPermittedError if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) if source_account.following?(target_account) # We're already following this account, but we'll call follow! again to @@ -32,7 +32,7 @@ class FollowService < BaseService if target_account.locked? || target_account.activitypub? request_follow(source_account, target_account, reblogs: reblogs) - else + elsif target_account.local? direct_follow(source_account, target_account, reblogs: reblogs) end end @@ -44,9 +44,6 @@ class FollowService < BaseService if target_account.local? LocalNotificationWorker.perform_async(target_account.id, follow_request.id, follow_request.class.name) - elsif target_account.ostatus? - NotificationWorker.perform_async(build_follow_request_xml(follow_request), source_account.id, target_account.id) - AfterRemoteFollowRequestWorker.perform_async(follow_request.id) elsif target_account.activitypub? ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), source_account.id, target_account.inbox_url) end @@ -57,27 +54,12 @@ class FollowService < BaseService def direct_follow(source_account, target_account, reblogs: true) follow = source_account.follow!(target_account, reblogs: reblogs) - if target_account.local? - LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name) - else - Pubsubhubbub::SubscribeWorker.perform_async(target_account.id) unless target_account.subscribed? - NotificationWorker.perform_async(build_follow_xml(follow), source_account.id, target_account.id) - AfterRemoteFollowWorker.perform_async(follow.id) - end - + LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name) MergeWorker.perform_async(target_account.id, source_account.id) follow end - def build_follow_request_xml(follow_request) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_request_salmon(follow_request)) - end - - def build_follow_xml(follow) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_salmon(follow)) - end - def build_json(follow_request) Oj.dump(serialize_payload(follow_request, ActivityPub::FollowSerializer)) end diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 7830aee11..34ec6d504 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -88,7 +88,6 @@ class PostStatusService < BaseService def postprocess_status! LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text? DistributionWorker.perform_async(@status.id) - Pubsubhubbub::DistributionWorker.perform_async(@status.stream_entry.id) ActivityPub::DistributionWorker.perform_async(@status.id) PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll end diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb deleted file mode 100644 index 30a9dd85e..000000000 --- a/app/services/process_feed_service.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class ProcessFeedService < BaseService - def call(body, account, **options) - @options = options - - xml = Nokogiri::XML(body) - xml.encoding = 'utf-8' - - update_author(body, account) - process_entries(xml, account) - end - - private - - def update_author(body, account) - RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true) - end - - def process_entries(xml, account) - xml.xpath('//xmlns:entry', xmlns: OStatus::TagManager::XMLNS).reverse_each.map { |entry| process_entry(entry, account) }.compact - end - - def process_entry(xml, account) - activity = OStatus::Activity::General.new(xml, account, @options) - activity.specialize&.perform if activity.status? - rescue ActiveRecord::RecordInvalid => e - Rails.logger.debug "Nothing was saved for #{activity.id} because: #{e}" - nil - end -end diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb deleted file mode 100644 index 1fca3832b..000000000 --- a/app/services/process_interaction_service.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -class ProcessInteractionService < BaseService - include AuthorExtractor - include Authorization - - # Record locally the remote interaction with our user - # @param [String] envelope Salmon envelope - # @param [Account] target_account Account the Salmon was addressed to - def call(envelope, target_account) - body = salmon.unpack(envelope) - - xml = Nokogiri::XML(body) - xml.encoding = 'utf-8' - - account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS)) - - return if account.nil? || account.suspended? - - if salmon.verify(envelope, account.keypair) - RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true) - - case verb(xml) - when :follow - follow!(account, target_account) unless target_account.locked? || target_account.blocking?(account) || target_account.domain_blocking?(account.domain) - when :request_friend - follow_request!(account, target_account) unless !target_account.locked? || target_account.blocking?(account) || target_account.domain_blocking?(account.domain) - when :authorize - authorize_follow_request!(account, target_account) - when :reject - reject_follow_request!(account, target_account) - when :unfollow - unfollow!(account, target_account) - when :favorite - favourite!(xml, account) - when :unfavorite - unfavourite!(xml, account) - when :post - add_post!(body, account) if mentions_account?(xml, target_account) - when :share - add_post!(body, account) unless status(xml).nil? - when :delete - delete_post!(xml, account) - when :block - reflect_block!(account, target_account) - when :unblock - reflect_unblock!(account, target_account) - end - end - rescue HTTP::Error, OStatus2::BadSalmonError, Mastodon::NotPermittedError - nil - end - - private - - def mentions_account?(xml, account) - xml.xpath('/xmlns:entry/xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each { |mention_link| return true if [OStatus::TagManager.instance.uri_for(account), OStatus::TagManager.instance.url_for(account)].include?(mention_link.attribute('href').value) } - false - end - - def verb(xml) - raw = xml.at_xpath('//activity:verb', activity: OStatus::TagManager::AS_XMLNS).content - OStatus::TagManager::VERBS.key(raw) - rescue - :post - end - - def follow!(account, target_account) - follow = account.follow!(target_account) - FollowRequest.find_by(account: account, target_account: target_account)&.destroy - NotifyService.new.call(target_account, follow) - end - - def follow_request!(account, target_account) - return if account.requested?(target_account) - - follow_request = FollowRequest.create!(account: account, target_account: target_account) - NotifyService.new.call(target_account, follow_request) - end - - def authorize_follow_request!(account, target_account) - follow_request = FollowRequest.find_by(account: target_account, target_account: account) - follow_request&.authorize! - Pubsubhubbub::SubscribeWorker.perform_async(account.id) unless account.subscribed? - end - - def reject_follow_request!(account, target_account) - follow_request = FollowRequest.find_by(account: target_account, target_account: account) - follow_request&.reject! - end - - def unfollow!(account, target_account) - account.unfollow!(target_account) - FollowRequest.find_by(account: account, target_account: target_account)&.destroy - end - - def reflect_block!(account, target_account) - UnfollowService.new.call(target_account, account) if target_account.following?(account) - account.block!(target_account) - end - - def reflect_unblock!(account, target_account) - UnblockService.new.call(account, target_account) - end - - def delete_post!(xml, account) - status = Status.find(xml.at_xpath('//xmlns:id', xmlns: OStatus::TagManager::XMLNS).content) - - return if status.nil? - - authorize_with account, status, :destroy? - - RemovalWorker.perform_async(status.id) - end - - def favourite!(xml, from_account) - current_status = status(xml) - - return if current_status.nil? - - favourite = current_status.favourites.where(account: from_account).first_or_create!(account: from_account) - NotifyService.new.call(current_status.account, favourite) - end - - def unfavourite!(xml, from_account) - current_status = status(xml) - - return if current_status.nil? - - favourite = current_status.favourites.where(account: from_account).first - favourite&.destroy - end - - def add_post!(body, account) - ProcessingWorker.perform_async(account.id, body.force_encoding('UTF-8')) - end - - def status(xml) - uri = activity_id(xml) - return nil unless OStatus::TagManager.instance.local_id?(uri) - Status.find(OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status')) - end - - def activity_id(xml) - xml.at_xpath('//activity:object', activity: OStatus::TagManager::AS_XMLNS).at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content - end - - def salmon - @salmon ||= OStatus2::Salmon.new - end -end diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index bc607dff3..da52bff6a 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class ProcessMentionsService < BaseService - include StreamEntryRenderer include Payloadable # Scan status for mentions and fetch remote mentioned users, create @@ -49,17 +48,11 @@ class ProcessMentionsService < BaseService if mentioned_account.local? LocalNotificationWorker.perform_async(mentioned_account.id, mention.id, mention.class.name) - elsif mentioned_account.ostatus? && !@status.stream_entry.hidden? - NotificationWorker.perform_async(ostatus_xml, @status.account_id, mentioned_account.id) elsif mentioned_account.activitypub? ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url) end end - def ostatus_xml - @ostatus_xml ||= stream_entry_to_xml(@status.stream_entry) - end - def activitypub_json return @activitypub_json if defined?(@activitypub_json) @activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account)) diff --git a/app/services/pubsubhubbub/subscribe_service.rb b/app/services/pubsubhubbub/subscribe_service.rb deleted file mode 100644 index 550da6328..000000000 --- a/app/services/pubsubhubbub/subscribe_service.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -class Pubsubhubbub::SubscribeService < BaseService - URL_PATTERN = /\A#{URI.regexp(%w(http https))}\z/ - - attr_reader :account, :callback, :secret, - :lease_seconds, :domain - - def call(account, callback, secret, lease_seconds, verified_domain = nil) - @account = account - @callback = Addressable::URI.parse(callback).normalize.to_s - @secret = secret - @lease_seconds = lease_seconds - @domain = verified_domain - - process_subscribe - end - - private - - def process_subscribe - if account.nil? - ['Invalid topic URL', 422] - elsif !valid_callback? - ['Invalid callback URL', 422] - elsif blocked_domain? - ['Callback URL not allowed', 403] - else - confirm_subscription - ['', 202] - end - end - - def confirm_subscription - subscription = locate_subscription - Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds) - end - - def valid_callback? - callback.present? && callback =~ URL_PATTERN - end - - def blocked_domain? - DomainBlock.blocked? Addressable::URI.parse(callback).host - end - - def locate_subscription - subscription = Subscription.find_or_initialize_by(account: account, callback_url: callback) - subscription.domain = domain - subscription.save! - subscription - end -end diff --git a/app/services/pubsubhubbub/unsubscribe_service.rb b/app/services/pubsubhubbub/unsubscribe_service.rb deleted file mode 100644 index 646150f7b..000000000 --- a/app/services/pubsubhubbub/unsubscribe_service.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class Pubsubhubbub::UnsubscribeService < BaseService - attr_reader :account, :callback - - def call(account, callback) - @account = account - @callback = Addressable::URI.parse(callback).normalize.to_s - - process_unsubscribe - end - - private - - def process_unsubscribe - if account.nil? - ['Invalid topic URL', 422] - else - confirm_unsubscribe unless subscription.nil? - ['', 202] - end - end - - def confirm_unsubscribe - Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'unsubscribe') - end - - def subscription - @_subscription ||= Subscription.find_by(account: account, callback_url: callback) - end -end diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 9cf4bc128..3bb460fca 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -2,7 +2,6 @@ class ReblogService < BaseService include Authorization - include StreamEntryRenderer include Payloadable # Reblog a status and notify its remote author @@ -24,7 +23,6 @@ class ReblogService < BaseService reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility) DistributionWorker.perform_async(reblog.id) - Pubsubhubbub::DistributionWorker.perform_async(reblog.stream_entry.id) ActivityPub::DistributionWorker.perform_async(reblog.id) create_notification(reblog) @@ -40,8 +38,6 @@ class ReblogService < BaseService if reblogged_status.account.local? LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name) - elsif reblogged_status.account.ostatus? - NotificationWorker.perform_async(stream_entry_to_xml(reblog.stream_entry), reblog.account_id, reblogged_status.account_id) elsif reblogged_status.account.activitypub? && !reblogged_status.account.following?(reblog.account) ActivityPub::DeliveryWorker.perform_async(build_json(reblog), reblog.account_id, reblogged_status.account.inbox_url) end diff --git a/app/services/reject_follow_service.rb b/app/services/reject_follow_service.rb index f87d0ba91..bc0000c8c 100644 --- a/app/services/reject_follow_service.rb +++ b/app/services/reject_follow_service.rb @@ -6,25 +6,17 @@ class RejectFollowService < BaseService def call(source_account, target_account) follow_request = FollowRequest.find_by!(account: source_account, target_account: target_account) follow_request.reject! - create_notification(follow_request) unless source_account.local? + create_notification(follow_request) if !source_account.local? && source_account.activitypub? follow_request end private def create_notification(follow_request) - if follow_request.account.ostatus? - NotificationWorker.perform_async(build_xml(follow_request), follow_request.target_account_id, follow_request.account_id) - elsif follow_request.account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url) - end + ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url) end def build_json(follow_request) Oj.dump(serialize_payload(follow_request, ActivityPub::RejectFollowSerializer)) end - - def build_xml(follow_request) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)) - end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 81adc5aae..a8c9100b3 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class RemoveStatusService < BaseService - include StreamEntryRenderer include Redisable include Payloadable @@ -78,11 +77,6 @@ class RemoveStatusService < BaseService target_accounts << @status.reblog.account if @status.reblog? && !@status.reblog.account.local? target_accounts.uniq!(&:id) - # Ostatus - NotificationWorker.push_bulk(target_accounts.select(&:ostatus?).uniq(&:domain)) do |target_account| - [salmon_xml, @account.id, target_account.id] - end - # ActivityPub ActivityPub::DeliveryWorker.push_bulk(target_accounts.select(&:activitypub?).uniq(&:preferred_inbox_url)) do |target_account| [signed_activity_json, @account.id, target_account.preferred_inbox_url] @@ -90,9 +84,6 @@ class RemoveStatusService < BaseService end def remove_from_remote_followers - # OStatus - Pubsubhubbub::RawDistributionWorker.perform_async(salmon_xml, @account.id) - # ActivityPub ActivityPub::DeliveryWorker.push_bulk(@account.followers.inboxes) do |inbox_url| [signed_activity_json, @account.id, inbox_url] @@ -111,10 +102,6 @@ class RemoveStatusService < BaseService end end - def salmon_xml - @salmon_xml ||= stream_entry_to_xml(@stream_entry) - end - def signed_activity_json @signed_activity_json ||= Oj.dump(serialize_payload(@status, @status.reblog? ? ActivityPub::UndoAnnounceSerializer : ActivityPub::DeleteSerializer, signer: @account)) end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index e557706da..0ea31a0d8 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true +require_relative '../models/account' + class ResolveAccountService < BaseService - include OStatus2::MagicKey include JsonLdHelper - DFRN_NS = 'http://purl.org/macgirvin/dfrn/1.0' - # Find or create a local account for a remote user. # When creating, look up the user's webfinger and fetch all # important information from their feed @@ -48,18 +47,16 @@ class ResolveAccountService < BaseService return end - return if links_missing? || auto_suspend? return Account.find_local(@username) if TagManager.instance.local_domain?(@domain) + return unless activitypub_ready? RedisLock.acquire(lock_options) do |lock| if lock.acquired? @account = Account.find_remote(@username, @domain) - if activitypub_ready? || @account&.activitypub? - handle_activitypub - else - handle_ostatus - end + next unless @account.nil? || @account.activitypub? + + handle_activitypub else raise Mastodon::RaceConditionError end @@ -73,38 +70,12 @@ class ResolveAccountService < BaseService private - def links_missing? - !(activitypub_ready? || ostatus_ready?) - end - - def ostatus_ready? - !(@webfinger.link('http://schemas.google.com/g/2010#updates-from').nil? || - @webfinger.link('salmon').nil? || - @webfinger.link('http://webfinger.net/rel/profile-page').nil? || - @webfinger.link('magic-public-key').nil? || - canonical_uri.nil? || - hub_url.nil?) - end - def webfinger_update_due? @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?) end def activitypub_ready? - !@webfinger.link('self').nil? && - ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) && - !actor_json.nil? && - actor_json['inbox'].present? - end - - def handle_ostatus - create_account if @account.nil? - update_account - update_account_profile if update_profile? - end - - def update_profile? - @options[:update_profile] + !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) end def handle_activitypub @@ -115,89 +86,10 @@ class ResolveAccountService < BaseService nil end - def create_account - Rails.logger.debug "Creating new remote account for #{@username}@#{@domain}" - - @account = Account.new(username: @username, domain: @domain) - @account.suspended_at = domain_block.created_at if auto_suspend? - @account.silenced_at = domain_block.created_at if auto_silence? - @account.private_key = nil - end - - def update_account - @account.last_webfingered_at = Time.now.utc - @account.protocol = :ostatus - @account.remote_url = atom_url - @account.salmon_url = salmon_url - @account.url = url - @account.public_key = public_key - @account.uri = canonical_uri - @account.hub_url = hub_url - @account.save! - end - - def auto_suspend? - domain_block&.suspend? - end - - def auto_silence? - domain_block&.silence? - end - - def domain_block - return @domain_block if defined?(@domain_block) - @domain_block = DomainBlock.rule_for(@domain) - end - - def atom_url - @atom_url ||= @webfinger.link('http://schemas.google.com/g/2010#updates-from').href - end - - def salmon_url - @salmon_url ||= @webfinger.link('salmon').href - end - def actor_url @actor_url ||= @webfinger.link('self').href end - def url - @url ||= @webfinger.link('http://webfinger.net/rel/profile-page').href - end - - def public_key - @public_key ||= magic_key_to_pem(@webfinger.link('magic-public-key').href) - end - - def canonical_uri - return @canonical_uri if defined?(@canonical_uri) - - author_uri = atom.at_xpath('/xmlns:feed/xmlns:author/xmlns:uri') - - if author_uri.nil? - owner = atom.at_xpath('/xmlns:feed').at_xpath('./dfrn:owner', dfrn: DFRN_NS) - author_uri = owner.at_xpath('./xmlns:uri') unless owner.nil? - end - - @canonical_uri = author_uri.nil? ? nil : author_uri.content - end - - def hub_url - return @hub_url if defined?(@hub_url) - - hubs = atom.xpath('//xmlns:link[@rel="hub"]') - @hub_url = hubs.empty? || hubs.first['href'].nil? ? nil : hubs.first['href'] - end - - def atom_body - return @atom_body if defined?(@atom_body) - - @atom_body = Request.new(:get, atom_url).perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response.code == 200 - response.body_with_limit - end - end - def actor_json return @actor_json if defined?(@actor_json) @@ -205,15 +97,6 @@ class ResolveAccountService < BaseService @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil end - def atom - return @atom if defined?(@atom) - @atom = Nokogiri::XML(atom_body) - end - - def update_account_profile - RemoteProfileUpdateWorker.perform_async(@account.id, atom_body.force_encoding('UTF-8'), false) - end - def lock_options { redis: Redis.current, key: "resolve:#{@username}@#{@domain}" } end diff --git a/app/services/send_interaction_service.rb b/app/services/send_interaction_service.rb deleted file mode 100644 index 3419043e5..000000000 --- a/app/services/send_interaction_service.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -class SendInteractionService < BaseService - # Send an Atom representation of an interaction to a remote Salmon endpoint - # @param [String] Entry XML - # @param [Account] source_account - # @param [Account] target_account - def call(xml, source_account, target_account) - @xml = xml - @source_account = source_account - @target_account = target_account - - return if !target_account.ostatus? || block_notification? - - build_request.perform do |delivery| - raise Mastodon::UnexpectedResponseError, delivery unless delivery.code > 199 && delivery.code < 300 - end - end - - private - - def build_request - request = Request.new(:post, @target_account.salmon_url, body: envelope) - request.add_headers('Content-Type' => 'application/magic-envelope+xml') - request - end - - def envelope - salmon.pack(@xml, @source_account.keypair) - end - - def block_notification? - DomainBlock.blocked?(@target_account.domain) - end - - def salmon - @salmon ||= OStatus2::Salmon.new - end -end diff --git a/app/services/subscribe_service.rb b/app/services/subscribe_service.rb deleted file mode 100644 index 83fd64396..000000000 --- a/app/services/subscribe_service.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -class SubscribeService < BaseService - def call(account) - return if account.hub_url.blank? - - @account = account - @account.secret = SecureRandom.hex - - build_request.perform do |response| - if response_failed_permanently? response - # We're not allowed to subscribe. Fail and move on. - @account.secret = '' - @account.save! - elsif response_successful? response - # The subscription will be confirmed asynchronously. - @account.save! - else - # The response was either a 429 rate limit, or a 5xx error. - # We need to retry at a later time. Fail loudly! - raise Mastodon::UnexpectedResponseError, response - end - end - end - - private - - def build_request - request = Request.new(:post, @account.hub_url, form: subscription_params) - request.on_behalf_of(some_local_account) if some_local_account - request - end - - def subscription_params - { - 'hub.topic': @account.remote_url, - 'hub.mode': 'subscribe', - 'hub.callback': api_subscription_url(@account.id), - 'hub.verify': 'async', - 'hub.secret': @account.secret, - 'hub.lease_seconds': 7.days.seconds, - } - end - - def some_local_account - @some_local_account ||= Account.local.without_suspended.first - end - - # Any response in the 3xx or 4xx range, except for 429 (rate limit) - def response_failed_permanently?(response) - (response.status.redirect? || response.status.client_error?) && !response.status.too_many_requests? - end - - # Any response in the 2xx range - def response_successful?(response) - response.status.success? - end -end diff --git a/app/services/unblock_service.rb b/app/services/unblock_service.rb index 95a858e9f..c263ac8af 100644 --- a/app/services/unblock_service.rb +++ b/app/services/unblock_service.rb @@ -7,25 +7,17 @@ class UnblockService < BaseService return unless account.blocking?(target_account) unblock = account.unblock!(target_account) - create_notification(unblock) unless target_account.local? + create_notification(unblock) if !target_account.local? && target_account.activitypub? unblock end private def create_notification(unblock) - if unblock.target_account.ostatus? - NotificationWorker.perform_async(build_xml(unblock), unblock.account_id, unblock.target_account_id) - elsif unblock.target_account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(unblock), unblock.account_id, unblock.target_account.inbox_url) - end + ActivityPub::DeliveryWorker.perform_async(build_json(unblock), unblock.account_id, unblock.target_account.inbox_url) end def build_json(unblock) Oj.dump(serialize_payload(unblock, ActivityPub::UndoBlockSerializer)) end - - def build_xml(block) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unblock_salmon(block)) - end end diff --git a/app/services/unfavourite_service.rb b/app/services/unfavourite_service.rb index dcc890b7d..37917a64f 100644 --- a/app/services/unfavourite_service.rb +++ b/app/services/unfavourite_service.rb @@ -6,7 +6,7 @@ class UnfavouriteService < BaseService def call(account, status) favourite = Favourite.find_by!(account: account, status: status) favourite.destroy! - create_notification(favourite) unless status.local? + create_notification(favourite) if !status.account.local? && status.account.activitypub? favourite end @@ -14,19 +14,10 @@ class UnfavouriteService < BaseService def create_notification(favourite) status = favourite.status - - if status.account.ostatus? - NotificationWorker.perform_async(build_xml(favourite), favourite.account_id, status.account_id) - elsif status.account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url) - end + ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url) end def build_json(favourite) Oj.dump(serialize_payload(favourite, ActivityPub::UndoLikeSerializer)) end - - def build_xml(favourite) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unfavourite_salmon(favourite)) - end end diff --git a/app/services/unfollow_service.rb b/app/services/unfollow_service.rb index 17dc29735..b7033d7eb 100644 --- a/app/services/unfollow_service.rb +++ b/app/services/unfollow_service.rb @@ -21,8 +21,8 @@ class UnfollowService < BaseService return unless follow follow.destroy! - create_notification(follow) unless @target_account.local? - create_reject_notification(follow) if @target_account.local? && !@source_account.local? + create_notification(follow) if !@target_account.local? && @target_account.activitypub? + create_reject_notification(follow) if @target_account.local? && !@source_account.local? && @source_account.activitypub? UnmergeWorker.perform_async(@target_account.id, @source_account.id) follow end @@ -38,16 +38,10 @@ class UnfollowService < BaseService end def create_notification(follow) - if follow.target_account.ostatus? - NotificationWorker.perform_async(build_xml(follow), follow.account_id, follow.target_account_id) - elsif follow.target_account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.account_id, follow.target_account.inbox_url) - end + ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.account_id, follow.target_account.inbox_url) end def create_reject_notification(follow) - # Rejecting an already-existing follow request - return unless follow.account.activitypub? ActivityPub::DeliveryWorker.perform_async(build_reject_json(follow), follow.target_account_id, follow.account.inbox_url) end @@ -58,8 +52,4 @@ class UnfollowService < BaseService def build_reject_json(follow) Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)) end - - def build_xml(follow) - OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unfollow_salmon(follow)) - end end diff --git a/app/services/unsubscribe_service.rb b/app/services/unsubscribe_service.rb deleted file mode 100644 index 95c1fb4fc..000000000 --- a/app/services/unsubscribe_service.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -class UnsubscribeService < BaseService - def call(account) - return if account.hub_url.blank? - - @account = account - - begin - build_request.perform do |response| - Rails.logger.debug "PuSH unsubscribe for #{@account.acct} failed: #{response.status}" unless response.status.success? - end - rescue HTTP::Error, OpenSSL::SSL::SSLError => e - Rails.logger.debug "PuSH unsubscribe for #{@account.acct} failed: #{e}" - end - - @account.secret = '' - @account.subscription_expires_at = nil - @account.save! - end - - private - - def build_request - Request.new(:post, @account.hub_url, form: subscription_params) - end - - def subscription_params - { - 'hub.topic': @account.remote_url, - 'hub.mode': 'unsubscribe', - 'hub.callback': api_subscription_url(@account.id), - 'hub.verify': 'async', - } - end -end diff --git a/app/services/update_remote_profile_service.rb b/app/services/update_remote_profile_service.rb deleted file mode 100644 index 403395a0d..000000000 --- a/app/services/update_remote_profile_service.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -class UpdateRemoteProfileService < BaseService - attr_reader :account, :remote_profile - - def call(body, account, resubscribe = false) - @account = account - @remote_profile = RemoteProfile.new(body) - - return if remote_profile.root.nil? - - update_account unless remote_profile.author.nil? - - old_hub_url = account.hub_url - account.hub_url = remote_profile.hub_link if remote_profile.hub_link.present? && remote_profile.hub_link != old_hub_url - - account.save_with_optional_media! - - Pubsubhubbub::SubscribeWorker.perform_async(account.id) if resubscribe && account.hub_url != old_hub_url - end - - private - - def update_account - account.display_name = remote_profile.display_name || '' - account.note = remote_profile.note || '' - account.locked = remote_profile.locked? - - if !account.suspended? && !DomainBlock.reject_media?(account.domain) - if remote_profile.avatar.present? - account.avatar_remote_url = remote_profile.avatar - else - account.avatar_remote_url = '' - account.avatar.destroy - end - - if remote_profile.header.present? - account.header_remote_url = remote_profile.header - else - account.header_remote_url = '' - account.header.destroy - end - - save_emojis if remote_profile.emojis.present? - end - end - - def save_emojis - do_not_download = DomainBlock.reject_media?(account.domain) - - return if do_not_download - - remote_profile.emojis.each do |link| - next unless link['href'] && link['name'] - - shortcode = link['name'].delete(':') - emoji = CustomEmoji.find_by(shortcode: shortcode, domain: account.domain) - - next unless emoji.nil? - - emoji = CustomEmoji.new(shortcode: shortcode, domain: account.domain) - emoji.image_remote_url = link['href'] - emoji.save - end - end -end diff --git a/app/services/verify_salmon_service.rb b/app/services/verify_salmon_service.rb deleted file mode 100644 index 205b35d8b..000000000 --- a/app/services/verify_salmon_service.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -class VerifySalmonService < BaseService - include AuthorExtractor - - def call(payload) - body = salmon.unpack(payload) - - xml = Nokogiri::XML(body) - xml.encoding = 'utf-8' - - account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS)) - - if account.nil? - false - else - salmon.verify(payload, account.keypair) - end - end - - private - - def salmon - @salmon ||= OStatus2::Salmon.new - end -end diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml index 950e61847..de7d2a8ba 100644 --- a/app/views/accounts/show.html.haml +++ b/app/views/accounts/show.html.haml @@ -7,7 +7,6 @@ - if @account.user&.setting_noindex %meta{ name: 'robots', content: 'noindex' }/ - %link{ rel: 'salmon', href: api_salmon_url(@account.id) }/ %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/ %link{ rel: 'alternate', type: 'application/rss+xml', href: account_url(@account, format: 'rss') }/ %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@account) }/ diff --git a/app/views/admin/subscriptions/_subscription.html.haml b/app/views/admin/subscriptions/_subscription.html.haml deleted file mode 100644 index 1dec8e396..000000000 --- a/app/views/admin/subscriptions/_subscription.html.haml +++ /dev/null @@ -1,18 +0,0 @@ -%tr - %td - %samp= subscription.account.acct - %td - %samp= subscription.callback_url - %td - - if subscription.confirmed? - %i.fa.fa-check - %td{ style: "color: #{subscription.expired? ? 'red' : 'inherit'};" } - %time.time-ago{ datetime: subscription.expires_at.iso8601, title: l(subscription.expires_at) } - = precede subscription.expired? ? '-' : '' do - = time_ago_in_words(subscription.expires_at) - %td - - if subscription.last_successful_delivery_at? - %time.formatted{ datetime: subscription.last_successful_delivery_at.iso8601, title: l(subscription.last_successful_delivery_at) } - = l subscription.last_successful_delivery_at - - else - %i.fa.fa-times diff --git a/app/views/admin/subscriptions/index.html.haml b/app/views/admin/subscriptions/index.html.haml deleted file mode 100644 index 83704c8ee..000000000 --- a/app/views/admin/subscriptions/index.html.haml +++ /dev/null @@ -1,16 +0,0 @@ -- content_for :page_title do - = t('admin.subscriptions.title') - -.table-wrapper - %table.table - %thead - %tr - %th= t('admin.subscriptions.topic') - %th= t('admin.subscriptions.callback_url') - %th= t('admin.subscriptions.confirmed') - %th= t('admin.subscriptions.expires_in') - %th= t('admin.subscriptions.last_delivery') - %tbody - = render @subscriptions - -= paginate @subscriptions diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby index 968c8c138..c82cdb7b3 100644 --- a/app/views/well_known/webfinger/show.xml.ruby +++ b/app/views/well_known/webfinger/show.xml.ruby @@ -25,11 +25,6 @@ doc << Ox::Element.new('XRD').tap do |xrd| link['href'] = account_url(@account) end - xrd << Ox::Element.new('Link').tap do |link| - link['rel'] = 'salmon' - link['href'] = api_salmon_url(@account.id) - end - xrd << Ox::Element.new('Link').tap do |link| link['rel'] = 'magic-public-key' link['href'] = "data:application/magic-public-key,#{@account.magic_key}" diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb index 84eb6ade2..ce9c65834 100644 --- a/app/workers/after_remote_follow_request_worker.rb +++ b/app/workers/after_remote_follow_request_worker.rb @@ -5,27 +5,5 @@ class AfterRemoteFollowRequestWorker sidekiq_options queue: 'pull', retry: 5 - attr_reader :follow_request - - def perform(follow_request_id) - @follow_request = FollowRequest.find(follow_request_id) - process_follow_service if processing_required? - rescue ActiveRecord::RecordNotFound - true - end - - private - - def process_follow_service - follow_request.destroy - FollowService.new.call(follow_request.account, updated_account.acct) - end - - def processing_required? - !updated_account.nil? && !updated_account.locked? - end - - def updated_account - @_updated_account ||= FetchRemoteAccountService.new.call(follow_request.target_account.remote_url) - end + def perform(follow_request_id); end end diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb index edab83f85..d9719f2bf 100644 --- a/app/workers/after_remote_follow_worker.rb +++ b/app/workers/after_remote_follow_worker.rb @@ -5,27 +5,5 @@ class AfterRemoteFollowWorker sidekiq_options queue: 'pull', retry: 5 - attr_reader :follow - - def perform(follow_id) - @follow = Follow.find(follow_id) - process_follow_service if processing_required? - rescue ActiveRecord::RecordNotFound - true - end - - private - - def process_follow_service - follow.destroy - FollowService.new.call(follow.account, updated_account.acct) - end - - def updated_account - @_updated_account ||= FetchRemoteAccountService.new.call(follow.target_account.remote_url) - end - - def processing_required? - !updated_account.nil? && updated_account.locked? - end + def perform(follow_id); end end diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb index da1d6ab45..1c0f001cf 100644 --- a/app/workers/notification_worker.rb +++ b/app/workers/notification_worker.rb @@ -5,7 +5,5 @@ class NotificationWorker sidekiq_options queue: 'push', retry: 5 - def perform(xml, source_account_id, target_account_id) - SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id)) - end + def perform(xml, source_account_id, target_account_id); end end diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb index 978c3aba2..cf3bd8397 100644 --- a/app/workers/processing_worker.rb +++ b/app/workers/processing_worker.rb @@ -5,7 +5,5 @@ class ProcessingWorker sidekiq_options backtrace: true - def perform(account_id, body) - ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true) - end + def perform(account_id, body); end end diff --git a/app/workers/pubsubhubbub/confirmation_worker.rb b/app/workers/pubsubhubbub/confirmation_worker.rb index c0e7b677e..783a8c95f 100644 --- a/app/workers/pubsubhubbub/confirmation_worker.rb +++ b/app/workers/pubsubhubbub/confirmation_worker.rb @@ -2,81 +2,8 @@ class Pubsubhubbub::ConfirmationWorker include Sidekiq::Worker - include RoutingHelper sidekiq_options queue: 'push', retry: false - attr_reader :subscription, :mode, :secret, :lease_seconds - - def perform(subscription_id, mode, secret = nil, lease_seconds = nil) - @subscription = Subscription.find(subscription_id) - @mode = mode - @secret = secret - @lease_seconds = lease_seconds - process_confirmation - end - - private - - def process_confirmation - prepare_subscription - - callback_get_with_params - logger.debug "Confirming PuSH subscription for #{subscription.callback_url} with challenge #{challenge}: #{@callback_response_body}" - - update_subscription - end - - def update_subscription - if successful_subscribe? - subscription.save! - elsif successful_unsubscribe? - subscription.destroy! - end - end - - def successful_subscribe? - subscribing? && response_matches_challenge? - end - - def successful_unsubscribe? - (unsubscribing? && response_matches_challenge?) || !subscription.confirmed? - end - - def response_matches_challenge? - @callback_response_body == challenge - end - - def subscribing? - mode == 'subscribe' - end - - def unsubscribing? - mode == 'unsubscribe' - end - - def callback_get_with_params - Request.new(:get, subscription.callback_url, params: callback_params).perform do |response| - @callback_response_body = response.body_with_limit - end - end - - def callback_params - { - 'hub.topic': account_url(subscription.account, format: :atom), - 'hub.mode': mode, - 'hub.challenge': challenge, - 'hub.lease_seconds': subscription.lease_seconds, - } - end - - def prepare_subscription - subscription.secret = secret - subscription.lease_seconds = lease_seconds - subscription.confirmed = true - end - - def challenge - @_challenge ||= SecureRandom.hex - end + def perform(subscription_id, mode, secret = nil, lease_seconds = nil); end end diff --git a/app/workers/pubsubhubbub/delivery_worker.rb b/app/workers/pubsubhubbub/delivery_worker.rb index 619bfa48a..1260060bd 100644 --- a/app/workers/pubsubhubbub/delivery_worker.rb +++ b/app/workers/pubsubhubbub/delivery_worker.rb @@ -2,80 +2,8 @@ class Pubsubhubbub::DeliveryWorker include Sidekiq::Worker - include RoutingHelper sidekiq_options queue: 'push', retry: 3, dead: false - sidekiq_retry_in do |count| - 5 * (count + 1) - end - - attr_reader :subscription, :payload - - def perform(subscription_id, payload) - @subscription = Subscription.find(subscription_id) - @payload = payload - process_delivery unless blocked_domain? - rescue => e - raise e.class, "Delivery failed for #{subscription&.callback_url}: #{e.message}", e.backtrace[0] - end - - private - - def process_delivery - callback_post_payload do |payload_delivery| - raise Mastodon::UnexpectedResponseError, payload_delivery unless response_successful? payload_delivery - end - - subscription.touch(:last_successful_delivery_at) - end - - def callback_post_payload(&block) - request = Request.new(:post, subscription.callback_url, body: payload) - request.add_headers(headers) - request.perform(&block) - end - - def blocked_domain? - DomainBlock.blocked?(host) - end - - def host - Addressable::URI.parse(subscription.callback_url).normalized_host - end - - def headers - { - 'Content-Type' => 'application/atom+xml', - 'Link' => link_header, - }.merge(signature_headers.to_h) - end - - def link_header - LinkHeader.new([hub_link_header, self_link_header]).to_s - end - - def hub_link_header - [api_push_url, [%w(rel hub)]] - end - - def self_link_header - [account_url(subscription.account, format: :atom), [%w(rel self)]] - end - - def signature_headers - { 'X-Hub-Signature' => payload_signature } if subscription.secret? - end - - def payload_signature - "sha1=#{hmac_payload_digest}" - end - - def hmac_payload_digest - OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), subscription.secret, payload) - end - - def response_successful?(payload_delivery) - payload_delivery.code > 199 && payload_delivery.code < 300 - end + def perform(subscription_id, payload); end end diff --git a/app/workers/pubsubhubbub/distribution_worker.rb b/app/workers/pubsubhubbub/distribution_worker.rb index fed5e917d..75bac5d6f 100644 --- a/app/workers/pubsubhubbub/distribution_worker.rb +++ b/app/workers/pubsubhubbub/distribution_worker.rb @@ -5,28 +5,5 @@ class Pubsubhubbub::DistributionWorker sidekiq_options queue: 'push' - def perform(stream_entry_ids) - stream_entries = StreamEntry.where(id: stream_entry_ids).includes(:status).reject { |e| e.status.nil? || e.status.hidden? } - - return if stream_entries.empty? - - @account = stream_entries.first.account - @subscriptions = active_subscriptions.to_a - - distribute_public!(stream_entries) - end - - private - - def distribute_public!(stream_entries) - @payload = OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, stream_entries)) - - Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription_id| - [subscription_id, @payload] - end - end - - def active_subscriptions - Subscription.where(account: @account).active.pluck(:id) - end + def perform(stream_entry_ids); end end diff --git a/app/workers/pubsubhubbub/raw_distribution_worker.rb b/app/workers/pubsubhubbub/raw_distribution_worker.rb index 16962a623..ece9c80ac 100644 --- a/app/workers/pubsubhubbub/raw_distribution_worker.rb +++ b/app/workers/pubsubhubbub/raw_distribution_worker.rb @@ -5,18 +5,5 @@ class Pubsubhubbub::RawDistributionWorker sidekiq_options queue: 'push' - def perform(xml, source_account_id) - @account = Account.find(source_account_id) - @subscriptions = active_subscriptions.to_a - - Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription| - [subscription.id, xml] - end - end - - private - - def active_subscriptions - Subscription.where(account: @account).active.select('id, callback_url, domain') - end + def perform(xml, source_account_id); end end diff --git a/app/workers/pubsubhubbub/subscribe_worker.rb b/app/workers/pubsubhubbub/subscribe_worker.rb index 2e176d1c1..b861b5e67 100644 --- a/app/workers/pubsubhubbub/subscribe_worker.rb +++ b/app/workers/pubsubhubbub/subscribe_worker.rb @@ -5,30 +5,5 @@ class Pubsubhubbub::SubscribeWorker sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false - sidekiq_retry_in do |count| - case count - when 0 - 30.minutes.seconds - when 1 - 2.hours.seconds - when 2 - 12.hours.seconds - else - 24.hours.seconds * (count - 2) - end - end - - sidekiq_retries_exhausted do |msg, _e| - account = Account.find(msg['args'].first) - Sidekiq.logger.error "PuSH subscription attempts for #{account.acct} exhausted. Unsubscribing" - ::UnsubscribeService.new.call(account) - end - - def perform(account_id) - account = Account.find(account_id) - logger.debug "PuSH re-subscribing to #{account.acct}" - ::SubscribeService.new.call(account) - rescue => e - raise e.class, "Subscribe failed for #{account&.acct}: #{e.message}", e.backtrace[0] - end + def perform(account_id); end end diff --git a/app/workers/pubsubhubbub/unsubscribe_worker.rb b/app/workers/pubsubhubbub/unsubscribe_worker.rb index a271715b7..0c1c263f6 100644 --- a/app/workers/pubsubhubbub/unsubscribe_worker.rb +++ b/app/workers/pubsubhubbub/unsubscribe_worker.rb @@ -5,11 +5,5 @@ class Pubsubhubbub::UnsubscribeWorker sidekiq_options queue: 'push', retry: false, unique: :until_executed, dead: false - def perform(account_id) - account = Account.find(account_id) - logger.debug "PuSH unsubscribing from #{account.acct}" - ::UnsubscribeService.new.call(account) - rescue ActiveRecord::RecordNotFound - true - end + def perform(account_id); end end diff --git a/app/workers/remote_profile_update_worker.rb b/app/workers/remote_profile_update_worker.rb index 03585ad2d..01e8daf8f 100644 --- a/app/workers/remote_profile_update_worker.rb +++ b/app/workers/remote_profile_update_worker.rb @@ -5,9 +5,5 @@ class RemoteProfileUpdateWorker sidekiq_options queue: 'pull' - def perform(account_id, body, resubscribe) - UpdateRemoteProfileService.new.call(body, Account.find(account_id), resubscribe) - rescue ActiveRecord::RecordNotFound - true - end + def perform(account_id, body, resubscribe); end end diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb index d37d40432..10200b06c 100644 --- a/app/workers/salmon_worker.rb +++ b/app/workers/salmon_worker.rb @@ -5,9 +5,5 @@ class SalmonWorker sidekiq_options backtrace: true - def perform(account_id, body) - ProcessInteractionService.new.call(body, Account.find(account_id)) - rescue Nokogiri::XML::XPath::SyntaxError, ActiveRecord::RecordNotFound - true - end + def perform(account_id, body); end end diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb index d5873bccb..6903cadc7 100644 --- a/app/workers/scheduler/subscriptions_scheduler.rb +++ b/app/workers/scheduler/subscriptions_scheduler.rb @@ -5,13 +5,5 @@ class Scheduler::SubscriptionsScheduler sidekiq_options unique: :until_executed, retry: 0 - def perform - Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id)) - end - - private - - def expiring_accounts - Account.expiring(1.day.from_now).partitioned - end + def perform; end end diff --git a/config/locales/en.yml b/config/locales/en.yml index d4f1855aa..611f36fdd 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -469,13 +469,6 @@ en: no_status_selected: No statuses were changed as none were selected title: Account statuses with_media: With media - subscriptions: - callback_url: Callback URL - confirmed: Confirmed - expires_in: Expires in - last_delivery: Last delivery - title: WebSub - topic: Topic tags: accounts: Accounts hidden: Hidden diff --git a/config/navigation.rb b/config/navigation.rb index df1024189..ef845d1fc 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -48,7 +48,6 @@ SimpleNavigation::Configuration.run do |navigation| s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), edit_admin_settings_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/settings} s.item :custom_emojis, safe_join([fa_icon('smile-o fw'), t('admin.custom_emojis.title')]), admin_custom_emojis_url, highlights_on: %r{/admin/custom_emojis} s.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/relays} - s.item :subscriptions, safe_join([fa_icon('paper-plane-o fw'), t('admin.subscriptions.title')]), admin_subscriptions_url, if: -> { current_user.admin? } s.item :sidekiq, safe_join([fa_icon('diamond fw'), 'Sidekiq']), sidekiq_url, link_html: { target: 'sidekiq' }, if: -> { current_user.admin? } s.item :pghero, safe_join([fa_icon('database fw'), 'PgHero']), pghero_url, link_html: { target: 'pghero' }, if: -> { current_user.admin? } end diff --git a/config/routes.rb b/config/routes.rb index 9ab5ba7f0..4b6d464c6 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -154,7 +154,6 @@ Rails.application.routes.draw do namespace :admin do get '/dashboard', to: 'dashboard#index' - resources :subscriptions, only: [:index] resources :domain_blocks, only: [:new, :create, :show, :destroy] resources :email_domain_blocks, only: [:index, :new, :create, :destroy] resources :action_logs, only: [:index] @@ -191,8 +190,6 @@ Rails.application.routes.draw do resources :accounts, only: [:index, :show] do member do - post :subscribe - post :unsubscribe post :enable post :unsilence post :unsuspend @@ -257,16 +254,6 @@ Rails.application.routes.draw do get '/admin', to: redirect('/admin/dashboard', status: 302) namespace :api do - # PubSubHubbub outgoing subscriptions - resources :subscriptions, only: [:show] - post '/subscriptions/:id', to: 'subscriptions#update' - - # PubSubHubbub incoming subscriptions - post '/push', to: 'push#update', as: :push - - # Salmon - post '/salmon/:id', to: 'salmon#update', as: :salmon - # OEmbed get '/oembed', to: 'oembed#show', as: :oembed @@ -318,7 +305,6 @@ Rails.application.routes.draw do get '/search', to: 'search#index', as: :search - resources :follows, only: [:create] resources :media, only: [:create, :update] resources :blocks, only: [:index] resources :mutes, only: [:index] diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 0ec1742ab..a16dea967 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -9,9 +9,6 @@ scheduled_statuses_scheduler: every: '5m' class: Scheduler::ScheduledStatusesScheduler - subscriptions_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *' - class: Scheduler::SubscriptionsScheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::MediaCleanupScheduler diff --git a/spec/controllers/admin/accounts_controller_spec.rb b/spec/controllers/admin/accounts_controller_spec.rb index a348ab3d7..608606ff9 100644 --- a/spec/controllers/admin/accounts_controller_spec.rb +++ b/spec/controllers/admin/accounts_controller_spec.rb @@ -75,44 +75,6 @@ RSpec.describe Admin::AccountsController, type: :controller do end end - describe 'POST #subscribe' do - subject { post :subscribe, params: { id: account.id } } - - let(:current_user) { Fabricate(:user, admin: admin) } - let(:account) { Fabricate(:account) } - - context 'when user is admin' do - let(:admin) { true } - - it { is_expected.to redirect_to admin_account_path(account.id) } - end - - context 'when user is not admin' do - let(:admin) { false } - - it { is_expected.to have_http_status :forbidden } - end - end - - describe 'POST #unsubscribe' do - subject { post :unsubscribe, params: { id: account.id } } - - let(:current_user) { Fabricate(:user, admin: admin) } - let(:account) { Fabricate(:account) } - - context 'when user is admin' do - let(:admin) { true } - - it { is_expected.to redirect_to admin_account_path(account.id) } - end - - context 'when user is not admin' do - let(:admin) { false } - - it { is_expected.to have_http_status :forbidden } - end - end - describe 'POST #memorialize' do subject { post :memorialize, params: { id: account.id } } diff --git a/spec/controllers/admin/subscriptions_controller_spec.rb b/spec/controllers/admin/subscriptions_controller_spec.rb deleted file mode 100644 index 967152abe..000000000 --- a/spec/controllers/admin/subscriptions_controller_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true -require 'rails_helper' - -RSpec.describe Admin::SubscriptionsController, type: :controller do - render_views - - describe 'GET #index' do - around do |example| - default_per_page = Subscription.default_per_page - Subscription.paginates_per 1 - example.run - Subscription.paginates_per default_per_page - end - - before do - sign_in Fabricate(:user, admin: true), scope: :user - end - - it 'renders subscriptions' do - Fabricate(:subscription) - specified = Fabricate(:subscription) - - get :index - - subscriptions = assigns(:subscriptions) - expect(subscriptions.count).to eq 1 - expect(subscriptions[0]).to eq specified - - expect(response).to have_http_status(200) - end - end -end diff --git a/spec/controllers/api/push_controller_spec.rb b/spec/controllers/api/push_controller_spec.rb deleted file mode 100644 index d769d8554..000000000 --- a/spec/controllers/api/push_controller_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -require 'rails_helper' - -RSpec.describe Api::PushController, type: :controller do - describe 'POST #update' do - context 'with hub.mode=subscribe' do - it 'creates a subscription' do - service = double(call: ['', 202]) - allow(Pubsubhubbub::SubscribeService).to receive(:new).and_return(service) - account = Fabricate(:account) - account_topic_url = "https://#{Rails.configuration.x.local_domain}/users/#{account.username}.atom" - post :update, params: { - 'hub.mode' => 'subscribe', - 'hub.topic' => account_topic_url, - 'hub.callback' => 'https://callback.host/api', - 'hub.lease_seconds' => '3600', - 'hub.secret' => 'as1234df', - } - - expect(service).to have_received(:call).with( - account, - 'https://callback.host/api', - 'as1234df', - '3600', - nil - ) - expect(response).to have_http_status(202) - end - end - - context 'with hub.mode=unsubscribe' do - it 'unsubscribes the account' do - service = double(call: ['', 202]) - allow(Pubsubhubbub::UnsubscribeService).to receive(:new).and_return(service) - account = Fabricate(:account) - account_topic_url = "https://#{Rails.configuration.x.local_domain}/users/#{account.username}.atom" - post :update, params: { - 'hub.mode' => 'unsubscribe', - 'hub.topic' => account_topic_url, - 'hub.callback' => 'https://callback.host/api', - } - - expect(service).to have_received(:call).with( - account, - 'https://callback.host/api', - ) - expect(response).to have_http_status(202) - end - end - - context 'with unknown mode' do - it 'returns an unknown mode error' do - post :update, params: { 'hub.mode' => 'fake' } - - expect(response).to have_http_status(422) - expect(response.body).to match(/Unknown mode/) - end - end - end -end diff --git a/spec/controllers/api/salmon_controller_spec.rb b/spec/controllers/api/salmon_controller_spec.rb deleted file mode 100644 index 235a29af0..000000000 --- a/spec/controllers/api/salmon_controller_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -require 'rails_helper' - -RSpec.describe Api::SalmonController, type: :controller do - render_views - - let(:account) { Fabricate(:user, account: Fabricate(:account, username: 'catsrgr8')).account } - - before do - stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt')) - stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt')) - stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - end - - describe 'POST #update' do - context 'with valid post data' do - before do - post :update, params: { id: account.id }, body: File.read(Rails.root.join('spec', 'fixtures', 'salmon', 'mention.xml')) - end - - it 'contains XML in the request body' do - expect(request.body.read).to be_a String - end - - it 'returns http success' do - expect(response).to have_http_status(202) - end - - it 'creates remote account' do - expect(Account.find_by(username: 'gargron', domain: 'quitter.no')).to_not be_nil - end - - it 'creates status' do - expect(Status.find_by(uri: 'tag:quitter.no,2016-03-20:noticeId=1276923:objectType=note')).to_not be_nil - end - - it 'creates mention for target account' do - expect(account.mentions.count).to eq 1 - end - end - - context 'with empty post data' do - before do - post :update, params: { id: account.id }, body: '' - end - - it 'returns http client error' do - expect(response).to have_http_status(400) - end - end - - context 'with invalid post data' do - before do - service = double(call: false) - allow(VerifySalmonService).to receive(:new).and_return(service) - - post :update, params: { id: account.id }, body: File.read(Rails.root.join('spec', 'fixtures', 'salmon', 'mention.xml')) - end - - it 'returns http client error' do - expect(response).to have_http_status(401) - end - end - end -end diff --git a/spec/controllers/api/subscriptions_controller_spec.rb b/spec/controllers/api/subscriptions_controller_spec.rb deleted file mode 100644 index 7a4252fe6..000000000 --- a/spec/controllers/api/subscriptions_controller_spec.rb +++ /dev/null @@ -1,68 +0,0 @@ -require 'rails_helper' - -RSpec.describe Api::SubscriptionsController, type: :controller do - render_views - - let(:account) { Fabricate(:account, username: 'gargron', domain: 'quitter.no', remote_url: 'topic_url', secret: 'abc') } - - describe 'GET #show' do - context 'with valid subscription' do - before do - get :show, params: { :id => account.id, 'hub.topic' => 'topic_url', 'hub.challenge' => '456', 'hub.lease_seconds' => "#{86400 * 30}" } - end - - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'echoes back the challenge' do - expect(response.body).to match '456' - end - end - - context 'with invalid subscription' do - before do - expect_any_instance_of(Account).to receive_message_chain(:subscription, :valid?).and_return(false) - get :show, params: { :id => account.id } - end - - it 'returns http success' do - expect(response).to have_http_status(404) - end - end - end - - describe 'POST #update' do - let(:feed) { File.read(Rails.root.join('spec', 'fixtures', 'push', 'feed.atom')) } - - before do - stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {}) - stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:get, "https://quitter.no/notice/1269244").to_return(status: 404) - stub_request(:get, "https://quitter.no/notice/1265331").to_return(status: 404) - stub_request(:get, "https://community.highlandarrow.com/notice/54411").to_return(status: 404) - stub_request(:get, "https://community.highlandarrow.com/notice/53857").to_return(status: 404) - stub_request(:get, "https://community.highlandarrow.com/notice/51852").to_return(status: 404) - stub_request(:get, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404) - stub_request(:get, "https://community.highlandarrow.com/notice/50467").to_return(status: 404) - stub_request(:get, "https://quitter.no/notice/1243309").to_return(status: 404) - stub_request(:get, "https://quitter.no/user/7477").to_return(status: 404) - stub_request(:any, "https://community.highlandarrow.com/user/1").to_return(status: 404) - stub_request(:any, "https://social.umeahackerspace.se/user/2").to_return(status: 404) - stub_request(:any, "https://gs.kawa-kun.com/user/2").to_return(status: 404) - stub_request(:any, "https://mastodon.social/users/Gargron").to_return(status: 404) - - request.env['HTTP_X_HUB_SIGNATURE'] = "sha1=#{OpenSSL::HMAC.hexdigest('sha1', 'abc', feed)}" - - post :update, params: { id: account.id }, body: feed - end - - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'creates statuses for feed' do - expect(account.statuses.count).to_not eq 0 - end - end -end diff --git a/spec/controllers/api/v1/follows_controller_spec.rb b/spec/controllers/api/v1/follows_controller_spec.rb deleted file mode 100644 index 089e0fe5e..000000000 --- a/spec/controllers/api/v1/follows_controller_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -require 'rails_helper' - -RSpec.describe Api::V1::FollowsController, type: :controller do - render_views - - let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:follows') } - - before do - allow(controller).to receive(:doorkeeper_token) { token } - end - - describe 'POST #create' do - before do - stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt')) - stub_request(:head, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(:status => 405, :body => "", :headers => {}) - stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt')) - stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {}) - stub_request(:post, "https://quitter.no/main/salmon/user/7477").to_return(:status => 200, :body => "", :headers => {}) - - post :create, params: { uri: 'gargron@quitter.no' } - end - - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'creates account for remote user' do - expect(Account.find_by(username: 'gargron', domain: 'quitter.no')).to_not be_nil - end - - it 'creates a follow relation between user and remote user' do - expect(user.account.following?(Account.find_by(username: 'gargron', domain: 'quitter.no'))).to be true - end - - it 'sends a salmon slap to the remote user' do - expect(a_request(:post, "https://quitter.no/main/salmon/user/7477")).to have_been_made - end - - it 'subscribes to remote hub' do - expect(a_request(:post, "https://quitter.no/main/push/hub")).to have_been_made - end - - it 'returns http success if already following, too' do - post :create, params: { uri: 'gargron@quitter.no' } - expect(response).to have_http_status(200) - end - end -end diff --git a/spec/fixtures/requests/webfinger.txt b/spec/fixtures/requests/webfinger.txt index edb8a2dbb..f337ecae6 100644 --- a/spec/fixtures/requests/webfinger.txt +++ b/spec/fixtures/requests/webfinger.txt @@ -8,4 +8,4 @@ Access-Control-Allow-Origin: * Vary: Accept-Encoding,Cookie Strict-Transport-Security: max-age=31536000; includeSubdomains; -{"subject":"acct:gargron@quitter.no","aliases":["https:\/\/quitter.no\/user\/7477","https:\/\/quitter.no\/gargron","https:\/\/quitter.no\/index.php\/user\/7477","https:\/\/quitter.no\/index.php\/gargron"],"links":[{"rel":"http:\/\/webfinger.net\/rel\/profile-page","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/gmpg.org\/xfn\/11","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"describedby","type":"application\/rdf+xml","href":"https:\/\/quitter.no\/gargron\/foaf"},{"rel":"http:\/\/apinamespace.org\/atom","type":"application\/atomsvc+xml","href":"https:\/\/quitter.no\/api\/statusnet\/app\/service\/gargron.xml"},{"rel":"http:\/\/apinamespace.org\/twitter","href":"https:\/\/quitter.no\/api\/"},{"rel":"http:\/\/specs.openid.net\/auth\/2.0\/provider","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/schemas.google.com\/g\/2010#updates-from","type":"application\/atom+xml","href":"https:\/\/quitter.no\/api\/statuses\/user_timeline\/7477.atom"},{"rel":"magic-public-key","href":"data:application\/magic-public-key,RSA.1ZBkHTavLvxH3FzlKv4O6WtlILKRFfNami3_Rcu8EuogtXSYiS-bB6hElZfUCSHbC4uLemOA34PEhz__CDMozax1iI_t8dzjDnh1x0iFSup7pSfW9iXk_WU3Dm74yWWW2jildY41vWgrEstuQ1dJ8vVFfSJ9T_tO4c-T9y8vDI8=.AQAB"},{"rel":"salmon","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-replies","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-mention","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/ostatus.org\/schema\/1.0\/subscribe","template":"https:\/\/quitter.no\/main\/ostatussub?profile={uri}"}]} \ No newline at end of file +{"subject":"acct:gargron@quitter.no","aliases":["https:\/\/quitter.no\/user\/7477","https:\/\/quitter.no\/gargron","https:\/\/quitter.no\/index.php\/user\/7477","https:\/\/quitter.no\/index.php\/gargron"],"links":[{"rel":"http:\/\/webfinger.net\/rel\/profile-page","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/gmpg.org\/xfn\/11","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"describedby","type":"application\/rdf+xml","href":"https:\/\/quitter.no\/gargron\/foaf"},{"rel":"http:\/\/apinamespace.org\/atom","type":"application\/atomsvc+xml","href":"https:\/\/quitter.no\/api\/statusnet\/app\/service\/gargron.xml"},{"rel":"http:\/\/apinamespace.org\/twitter","href":"https:\/\/quitter.no\/api\/"},{"rel":"http:\/\/specs.openid.net\/auth\/2.0\/provider","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/schemas.google.com\/g\/2010#updates-from","type":"application\/atom+xml","href":"https:\/\/quitter.no\/api\/statuses\/user_timeline\/7477.atom"},{"rel":"magic-public-key","href":"data:application\/magic-public-key,RSA.1ZBkHTavLvxH3FzlKv4O6WtlILKRFfNami3_Rcu8EuogtXSYiS-bB6hElZfUCSHbC4uLemOA34PEhz__CDMozax1iI_t8dzjDnh1x0iFSup7pSfW9iXk_WU3Dm74yWWW2jildY41vWgrEstuQ1dJ8vVFfSJ9T_tO4c-T9y8vDI8=.AQAB"},{"rel":"salmon","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-replies","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-mention","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/ostatus.org\/schema\/1.0\/subscribe","template":"https:\/\/quitter.no\/main\/ostatussub?profile={uri}"}]} diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb index 891871c1c..74ab7576f 100644 --- a/spec/lib/ostatus/atom_serializer_spec.rb +++ b/spec/lib/ostatus/atom_serializer_spec.rb @@ -406,28 +406,6 @@ RSpec.describe OStatus::AtomSerializer do scope = entry.nodes.find { |node| node.name == 'mastodon:scope' } expect(scope.text).to eq 'public' end - - it 'returns element whose rendered view triggers creation when processed' do - remote_account = Account.create!(username: 'username') - remote_status = Fabricate(:status, account: remote_account, created_at: '2000-01-01T00:00:00Z') - - entry = OStatus::AtomSerializer.new.entry(remote_status.stream_entry, true) - entry.nodes.delete_if { |node| node[:type] == 'application/activity+json' } # Remove ActivityPub link to simplify test - xml = OStatus::AtomSerializer.render(entry).gsub('cb6e6126.ngrok.io', 'remote.test') - - remote_status.destroy! - remote_account.destroy! - - account = Account.create!( - domain: 'remote.test', - username: 'username', - last_webfingered_at: Time.now.utc - ) - - ProcessFeedService.new.call(xml, account) - - expect(Status.find_by(uri: "https://remote.test/users/#{remote_status.account.to_param}/statuses/#{remote_status.id}")).to be_instance_of Status - end end context 'if status is not present' do @@ -683,24 +661,6 @@ RSpec.describe OStatus::AtomSerializer do end end - it 'appends link element for hub' do - account = Fabricate(:account, username: 'username') - - feed = OStatus::AtomSerializer.new.feed(account, []) - - link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'hub' } - expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/api/push' - end - - it 'appends link element for Salmon' do - account = Fabricate(:account, username: 'username') - - feed = OStatus::AtomSerializer.new.feed(account, []) - - link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'salmon' } - expect(link[:href]).to start_with 'https://cb6e6126.ngrok.io/api/salmon/' - end - it 'appends stream entries' do account = Fabricate(:account, username: 'username') status = Fabricate(:status, account: account) @@ -784,18 +744,6 @@ RSpec.describe OStatus::AtomSerializer do object = block_salmon.nodes.find { |node| node.name == 'activity:object' } expect(object.id.text).to eq 'https://domain.test/id' end - - it 'returns element whose rendered view triggers block when processed' do - block = Fabricate(:block) - block_salmon = OStatus::AtomSerializer.new.block_salmon(block) - xml = OStatus::AtomSerializer.render(block_salmon) - envelope = OStatus2::Salmon.new.pack(xml, block.account.keypair) - block.destroy! - - ProcessInteractionService.new.call(envelope, block.target_account) - - expect(block.account.blocking?(block.target_account)).to be true - end end describe '#unblock_salmon' do @@ -871,17 +819,6 @@ RSpec.describe OStatus::AtomSerializer do object = unblock_salmon.nodes.find { |node| node.name == 'activity:object' } expect(object.id.text).to eq 'https://domain.test/id' end - - it 'returns element whose rendered view triggers block when processed' do - block = Fabricate(:block) - unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block) - xml = OStatus::AtomSerializer.render(unblock_salmon) - envelope = OStatus2::Salmon.new.pack(xml, block.account.keypair) - - ProcessInteractionService.new.call(envelope, block.target_account) - - expect { block.reload }.to raise_error ActiveRecord::RecordNotFound - end end describe '#favourite_salmon' do @@ -964,17 +901,6 @@ RSpec.describe OStatus::AtomSerializer do expect(favourite_salmon.title.text).to eq 'account favourited a status by status_account@remote' expect(favourite_salmon.content.text).to eq 'account favourited a status by status_account@remote' end - - it 'returns element whose rendered view triggers favourite when processed' do - favourite = Fabricate(:favourite) - favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite) - xml = OStatus::AtomSerializer.render(favourite_salmon) - envelope = OStatus2::Salmon.new.pack(xml, favourite.account.keypair) - favourite.destroy! - - ProcessInteractionService.new.call(envelope, favourite.status.account) - expect(favourite.account.favourited?(favourite.status)).to be true - end end describe '#unfavourite_salmon' do @@ -1064,16 +990,6 @@ RSpec.describe OStatus::AtomSerializer do expect(unfavourite_salmon.title.text).to eq 'account no longer favourites a status by status_account@remote' expect(unfavourite_salmon.content.text).to eq 'account no longer favourites a status by status_account@remote' end - - it 'returns element whose rendered view triggers unfavourite when processed' do - favourite = Fabricate(:favourite) - unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite) - xml = OStatus::AtomSerializer.render(unfavourite_salmon) - envelope = OStatus2::Salmon.new.pack(xml, favourite.account.keypair) - - ProcessInteractionService.new.call(envelope, favourite.status.account) - expect { favourite.reload }.to raise_error ActiveRecord::RecordNotFound - end end describe '#follow_salmon' do @@ -1143,18 +1059,6 @@ RSpec.describe OStatus::AtomSerializer do expect(follow_salmon.title.text).to eq 'account started following target_account@remote' expect(follow_salmon.content.text).to eq 'account started following target_account@remote' end - - it 'returns element whose rendered view triggers follow when processed' do - follow = Fabricate(:follow) - follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow) - xml = OStatus::AtomSerializer.render(follow_salmon) - follow.destroy! - envelope = OStatus2::Salmon.new.pack(xml, follow.account.keypair) - - ProcessInteractionService.new.call(envelope, follow.target_account) - - expect(follow.account.following?(follow.target_account)).to be true - end end describe '#unfollow_salmon' do @@ -1251,19 +1155,6 @@ RSpec.describe OStatus::AtomSerializer do object = unfollow_salmon.nodes.find { |node| node.name == 'activity:object' } expect(object.id.text).to eq 'https://domain.test/id' end - - it 'returns element whose rendered view triggers unfollow when processed' do - follow = Fabricate(:follow) - follow.destroy! - unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow) - xml = OStatus::AtomSerializer.render(unfollow_salmon) - follow.account.follow!(follow.target_account) - envelope = OStatus2::Salmon.new.pack(xml, follow.account.keypair) - - ProcessInteractionService.new.call(envelope, follow.target_account) - - expect(follow.account.following?(follow.target_account)).to be false - end end describe '#follow_request_salmon' do @@ -1294,18 +1185,6 @@ RSpec.describe OStatus::AtomSerializer do follow_request_salmon = serialize(follow_request) expect(follow_request_salmon.title.text).to eq 'account requested to follow target_account@remote' end - - it 'returns element whose rendered view triggers follow request when processed' do - follow_request = Fabricate(:follow_request) - follow_request_salmon = serialize(follow_request) - xml = OStatus::AtomSerializer.render(follow_request_salmon) - envelope = OStatus2::Salmon.new.pack(xml, follow_request.account.keypair) - follow_request.destroy! - - ProcessInteractionService.new.call(envelope, follow_request.target_account) - - expect(follow_request.account.requested?(follow_request.target_account)).to eq true - end end end @@ -1364,18 +1243,6 @@ RSpec.describe OStatus::AtomSerializer do verb = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' } expect(verb.text).to eq OStatus::TagManager::VERBS[:authorize] end - - it 'returns element whose rendered view creates follow from follow request when processed' do - follow_request = Fabricate(:follow_request) - authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request) - xml = OStatus::AtomSerializer.render(authorize_follow_request_salmon) - envelope = OStatus2::Salmon.new.pack(xml, follow_request.target_account.keypair) - - ProcessInteractionService.new.call(envelope, follow_request.account) - - expect(follow_request.account.following?(follow_request.target_account)).to eq true - expect { follow_request.reload }.to raise_error ActiveRecord::RecordNotFound - end end describe '#reject_follow_request_salmon' do @@ -1427,18 +1294,6 @@ RSpec.describe OStatus::AtomSerializer do verb = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' } expect(verb.text).to eq OStatus::TagManager::VERBS[:reject] end - - it 'returns element whose rendered view deletes follow request when processed' do - follow_request = Fabricate(:follow_request) - reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request) - xml = OStatus::AtomSerializer.render(reject_follow_request_salmon) - envelope = OStatus2::Salmon.new.pack(xml, follow_request.target_account.keypair) - - ProcessInteractionService.new.call(envelope, follow_request.account) - - expect(follow_request.account.following?(follow_request.target_account)).to eq false - expect { follow_request.reload }.to raise_error ActiveRecord::RecordNotFound - end end describe '#object' do diff --git a/spec/services/authorize_follow_service_spec.rb b/spec/services/authorize_follow_service_spec.rb index 562ef0041..ce56d57a6 100644 --- a/spec/services/authorize_follow_service_spec.rb +++ b/spec/services/authorize_follow_service_spec.rb @@ -38,13 +38,6 @@ RSpec.describe AuthorizeFollowService, type: :service do it 'creates follow relation' do expect(bob.following?(sender)).to be true end - - it 'sends a follow request authorization salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:authorize]) - }).to have_been_made.once - end end describe 'remote ActivityPub' do diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb index e53623449..d52e7f484 100644 --- a/spec/services/batched_remove_status_service_spec.rb +++ b/spec/services/batched_remove_status_service_spec.rb @@ -49,19 +49,6 @@ RSpec.describe BatchedRemoveStatusService, type: :service do expect(Redis.current).to have_received(:publish).with('timeline:public', any_args).at_least(:once) end - it 'sends PuSH update to PuSH subscribers' do - expect(a_request(:post, 'http://example.com/push').with { |req| - matches = req.body.match(OStatus::TagManager::VERBS[:delete]) - }).to have_been_made.at_least_once - end - - it 'sends Salmon slap to previously mentioned users' do - expect(a_request(:post, "http://example.com/salmon").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:delete]) - }).to have_been_made.once - end - it 'sends delete activity to followers' do expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.at_least_once end diff --git a/spec/services/block_service_spec.rb b/spec/services/block_service_spec.rb index 6584bb90e..de20dd026 100644 --- a/spec/services/block_service_spec.rb +++ b/spec/services/block_service_spec.rb @@ -28,13 +28,6 @@ RSpec.describe BlockService, type: :service do it 'creates a blocking relation' do expect(sender.blocking?(bob)).to be true end - - it 'sends a block salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:block]) - }).to have_been_made.once - end end describe 'remote ActivityPub' do diff --git a/spec/services/favourite_service_spec.rb b/spec/services/favourite_service_spec.rb index 0a20ccf6e..4c29ea77b 100644 --- a/spec/services/favourite_service_spec.rb +++ b/spec/services/favourite_service_spec.rb @@ -30,13 +30,6 @@ RSpec.describe FavouriteService, type: :service do it 'creates a favourite' do expect(status.favourites.first).to_not be_nil end - - it 'sends a salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:favorite]) - }).to have_been_made.once - end end describe 'remote ActivityPub' do diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb index 3cd86708b..37e9910d4 100644 --- a/spec/services/fetch_remote_account_service_spec.rb +++ b/spec/services/fetch_remote_account_service_spec.rb @@ -36,36 +36,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do include_examples 'return Account' end - context 'protocol is :ostatus' do - let(:prefetched_body) { xml } - let(:protocol) { :ostatus } - - before do - stub_request(:get, "https://kickass.zone/.well-known/webfinger?resource=acct:localhost@kickass.zone").to_return(request_fixture('webfinger-hacker3.txt')) - stub_request(:get, "https://kickass.zone/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt')) - end - - include_examples 'return Account' - - it 'does not update account information if XML comes from an unverified domain' do - feed_xml = <<-XML.squish - - - - http://activitystrea.ms/schema/1.0/person - http://kickass.zone/users/localhost - localhost - localhost - Villain!!! - - - XML - - returned_account = described_class.new.call('https://real-fake-domains.com/alice', feed_xml, :ostatus) - expect(returned_account.display_name).to_not eq 'Villain!!!' - end - end - context 'when prefetched_body is nil' do context 'protocol is :activitypub' do before do @@ -75,15 +45,5 @@ RSpec.describe FetchRemoteAccountService, type: :service do include_examples 'return Account' end - - context 'protocol is :ostatus' do - before do - stub_request(:get, url).to_return(status: 200, body: xml, headers: { 'Content-Type' => 'application/atom+xml' }) - stub_request(:get, "https://kickass.zone/.well-known/webfinger?resource=acct:localhost@kickass.zone").to_return(request_fixture('webfinger-hacker3.txt')) - stub_request(:get, "https://kickass.zone/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt')) - end - - include_examples 'return Account' - end end end diff --git a/spec/services/follow_service_spec.rb b/spec/services/follow_service_spec.rb index 3c4ec59be..86c85293e 100644 --- a/spec/services/follow_service_spec.rb +++ b/spec/services/follow_service_spec.rb @@ -96,74 +96,6 @@ RSpec.describe FollowService, type: :service do end end - context 'remote OStatus account' do - describe 'locked account' do - let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, locked: true, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com')).account } - - before do - stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {}) - subject.call(sender, bob.acct) - end - - it 'creates a follow request' do - expect(FollowRequest.find_by(account: sender, target_account: bob)).to_not be_nil - end - - it 'sends a follow request salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:request_friend]) - }).to have_been_made.once - end - end - - describe 'unlocked account' do - let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account } - - before do - stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {}) - stub_request(:post, "http://hub.example.com/").to_return(status: 202) - subject.call(sender, bob.acct) - end - - it 'creates a following relation' do - expect(sender.following?(bob)).to be true - end - - it 'sends a follow salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:follow]) - }).to have_been_made.once - end - - it 'subscribes to PuSH' do - expect(a_request(:post, "http://hub.example.com/")).to have_been_made.once - end - end - - describe 'already followed account' do - let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account } - - before do - sender.follow!(bob) - subject.call(sender, bob.acct) - end - - it 'keeps a following relation' do - expect(sender.following?(bob)).to be true - end - - it 'does not send a follow salmon slap' do - expect(a_request(:post, "http://salmon.example.com/")).not_to have_been_made - end - - it 'does not subscribe to PuSH' do - expect(a_request(:post, "http://hub.example.com/")).not_to have_been_made - end - end - end - context 'remote ActivityPub account' do let(:bob) { Fabricate(:user, account: Fabricate(:account, username: 'bob', domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox')).account } diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb index 5cf2dadf0..5355133f4 100644 --- a/spec/services/import_service_spec.rb +++ b/spec/services/import_service_spec.rb @@ -3,7 +3,11 @@ require 'rails_helper' RSpec.describe ImportService, type: :service do let!(:account) { Fabricate(:account, locked: false) } let!(:bob) { Fabricate(:account, username: 'bob', locked: false) } - let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false) } + let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') } + + before do + stub_request(:post, "https://example.com/inbox").to_return(status: 200) + end context 'import old-style list of muted users' do subject { ImportService.new } @@ -95,7 +99,8 @@ RSpec.describe ImportService, type: :service do let(:import) { Import.create(account: account, type: 'following', data: csv) } it 'follows the listed accounts, including boosts' do subject.call(import) - expect(account.following.count).to eq 2 + expect(account.following.count).to eq 1 + expect(account.follow_requests.count).to eq 1 expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true end end @@ -106,7 +111,8 @@ RSpec.describe ImportService, type: :service do it 'follows the listed accounts, including notifications' do account.follow!(bob, reblogs: false) subject.call(import) - expect(account.following.count).to eq 2 + expect(account.following.count).to eq 1 + expect(account.follow_requests.count).to eq 1 expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true end end @@ -117,7 +123,8 @@ RSpec.describe ImportService, type: :service do it 'mutes the listed accounts, including notifications' do account.follow!(bob, reblogs: false) subject.call(import) - expect(account.following.count).to eq 2 + expect(account.following.count).to eq 1 + expect(account.follow_requests.count).to eq 1 expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true end end @@ -136,9 +143,10 @@ RSpec.describe ImportService, type: :service do let(:import) { Import.create(account: account, type: 'following', data: csv) } it 'follows the listed accounts, respecting boosts' do subject.call(import) - expect(account.following.count).to eq 2 + expect(account.following.count).to eq 1 + expect(account.follow_requests.count).to eq 1 expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false + expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false end end @@ -148,9 +156,10 @@ RSpec.describe ImportService, type: :service do it 'mutes the listed accounts, respecting notifications' do account.follow!(bob, reblogs: true) subject.call(import) - expect(account.following.count).to eq 2 + expect(account.following.count).to eq 1 + expect(account.follow_requests.count).to eq 1 expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false + expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false end end @@ -160,9 +169,10 @@ RSpec.describe ImportService, type: :service do it 'mutes the listed accounts, respecting notifications' do account.follow!(bob, reblogs: true) subject.call(import) - expect(account.following.count).to eq 2 + expect(account.following.count).to eq 1 + expect(account.follow_requests.count).to eq 1 expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true - expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false + expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false end end end diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index facbe977f..bf06f50e9 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -144,7 +144,6 @@ RSpec.describe PostStatusService, type: :service do it 'gets distributed' do allow(DistributionWorker).to receive(:perform_async) - allow(Pubsubhubbub::DistributionWorker).to receive(:perform_async) allow(ActivityPub::DistributionWorker).to receive(:perform_async) account = Fabricate(:account) @@ -152,7 +151,6 @@ RSpec.describe PostStatusService, type: :service do status = subject.call(account, text: "test status update") expect(DistributionWorker).to have_received(:perform_async).with(status.id) - expect(Pubsubhubbub::DistributionWorker).to have_received(:perform_async).with(status.stream_entry.id) expect(ActivityPub::DistributionWorker).to have_received(:perform_async).with(status.id) end diff --git a/spec/services/process_feed_service_spec.rb b/spec/services/process_feed_service_spec.rb deleted file mode 100644 index 9d3465f3f..000000000 --- a/spec/services/process_feed_service_spec.rb +++ /dev/null @@ -1,252 +0,0 @@ -require 'rails_helper' - -RSpec.describe ProcessFeedService, type: :service do - subject { ProcessFeedService.new } - - describe 'processing a feed' do - let(:body) { File.read(Rails.root.join('spec', 'fixtures', 'xml', 'mastodon.atom')) } - let(:account) { Fabricate(:account, username: 'localhost', domain: 'kickass.zone') } - - before do - stub_request(:post, "https://pubsubhubbub.superfeedr.com/").to_return(:status => 200, :body => "", :headers => {}) - stub_request(:head, "http://kickass.zone/media/2").to_return(:status => 404) - stub_request(:head, "http://kickass.zone/media/3").to_return(:status => 404) - stub_request(:get, "http://kickass.zone/system/accounts/avatars/000/000/001/large/eris.png").to_return(request_fixture('avatar.txt')) - stub_request(:get, "http://kickass.zone/system/media_attachments/files/000/000/002/original/morpheus_linux.jpg?1476059910").to_return(request_fixture('attachment1.txt')) - stub_request(:get, "http://kickass.zone/system/media_attachments/files/000/000/003/original/gizmo.jpg?1476060065").to_return(request_fixture('attachment2.txt')) - end - - context 'when domain does not reject media' do - before do - subject.call(body, account) - end - - it 'updates remote user\'s account information' do - account.reload - expect(account.display_name).to eq '::1' - expect(account).to have_attached_file(:avatar) - expect(account.avatar_file_name).not_to be_nil - end - - it 'creates posts' do - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Status')).to_not be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')).to_not be_nil - end - - it 'marks replies as replies' do - status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status') - expect(status.reply?).to be true - end - - it 'sets account being replied to when possible' do - status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status') - expect(status.in_reply_to_account_id).to eq status.account_id - end - - it 'ignores delete statuses unless they existed before' do - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=3:objectType=Status')).to be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=12:objectType=Status')).to be_nil - end - - it 'does not create statuses for follows' do - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Follow')).to be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Follow')).to be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=4:objectType=Follow')).to be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=7:objectType=Follow')).to be_nil - end - - it 'does not create statuses for favourites' do - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Favourite')).to be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=3:objectType=Favourite')).to be_nil - end - - it 'creates posts with media' do - status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=14:objectType=Status') - - expect(status).to_not be_nil - expect(status.media_attachments.first).to have_attached_file(:file) - expect(status.media_attachments.first.image?).to be true - expect(status.media_attachments.first.file_file_name).not_to be_nil - end - end - - context 'when domain is set to reject media' do - let!(:domain_block) { Fabricate(:domain_block, domain: 'kickass.zone', reject_media: true) } - - before do - subject.call(body, account) - end - - it 'updates remote user\'s account information' do - account.reload - expect(account.display_name).to eq '::1' - end - - it 'rejects remote user\'s avatar' do - account.reload - expect(account.display_name).to eq '::1' - expect(account.avatar_file_name).to be_nil - end - - it 'creates posts' do - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Status')).to_not be_nil - expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')).to_not be_nil - end - - it 'creates posts with remote-only media' do - status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=14:objectType=Status') - - expect(status).to_not be_nil - expect(status.media_attachments.first.file_file_name).to be_nil - expect(status.media_attachments.first.unknown?).to be true - end - end - end - - it 'does not accept tampered reblogs' do - good_actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com') - - real_body = < - - tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status - 2017-04-27T13:49:25Z - 2017-04-27T13:49:25Z - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - https://overwatch.com/users/tracer - http://activitystrea.ms/schema/1.0/person - https://overwatch.com/users/tracer - tracer - - Overwatch rocks - -XML - - stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body, headers: { 'Content-Type' => 'application/atom+xml' }) - - bad_actor = Fabricate(:account, username: 'sombra', domain: 'talon.xyz') - - body = < - - tag:talon.xyz,2017-04-27:objectId=4467137:objectType=Status - 2017-04-27T13:49:25Z - 2017-04-27T13:49:25Z - - https://talon.xyz/users/sombra - http://activitystrea.ms/schema/1.0/person - https://talon.xyz/users/sombra - sombra - - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - Overwatch SUCKS AHAHA - - tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - https://overwatch.com/users/tracer - http://activitystrea.ms/schema/1.0/person - https://overwatch.com/users/tracer - tracer - - Overwatch SUCKS AHAHA - - - -XML - created_statuses = subject.call(body, bad_actor) - - expect(created_statuses.first.reblog?).to be true - expect(created_statuses.first.account_id).to eq bad_actor.id - expect(created_statuses.first.reblog.account_id).to eq good_actor.id - expect(created_statuses.first.reblog.text).to eq 'Overwatch rocks' - end - - it 'ignores reblogs if it failed to retrieve reblogged statuses' do - stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404) - - actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com') - - body = < - - tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status - 2017-04-27T13:49:25Z - 2017-04-27T13:49:25Z - - https://overwatch.com/users/tracer - http://activitystrea.ms/schema/1.0/person - https://overwatch.com/users/tracer - tracer - - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - Overwatch rocks - - tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - https://overwatch.com/users/tracer - http://activitystrea.ms/schema/1.0/person - https://overwatch.com/users/tracer - tracer - - Overwatch rocks - - -XML - - created_statuses = subject.call(body, actor) - - expect(created_statuses).to eq [] - end - - it 'ignores statuses with an out-of-order delete' do - sender = Fabricate(:account, username: 'tracer', domain: 'overwatch.com') - - delete_body = < - - tag:overwatch.com,2017-04-27:objectId=4487555:objectType=Status - 2017-04-27T13:49:25Z - 2017-04-27T13:49:25Z - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/delete - - https://overwatch.com/users/tracer - http://activitystrea.ms/schema/1.0/person - https://overwatch.com/users/tracer - tracer - - -XML - - status_body = < - - tag:overwatch.com,2017-04-27:objectId=4487555:objectType=Status - 2017-04-27T13:49:25Z - 2017-04-27T13:49:25Z - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - https://overwatch.com/users/tracer - http://activitystrea.ms/schema/1.0/person - https://overwatch.com/users/tracer - tracer - - Overwatch rocks - -XML - - subject.call(delete_body, sender) - created_statuses = subject.call(status_body, sender) - - expect(created_statuses).to be_empty - end -end diff --git a/spec/services/process_interaction_service_spec.rb b/spec/services/process_interaction_service_spec.rb deleted file mode 100644 index b858c19d0..000000000 --- a/spec/services/process_interaction_service_spec.rb +++ /dev/null @@ -1,151 +0,0 @@ -require 'rails_helper' - -RSpec.describe ProcessInteractionService, type: :service do - let(:receiver) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account } - let(:sender) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } - let(:remote_sender) { Fabricate(:account, username: 'carol', domain: 'localdomain.com', uri: 'https://webdomain.com/users/carol') } - - subject { ProcessInteractionService.new } - - describe 'status delete slap' do - let(:remote_status) { Fabricate(:status, account: remote_sender) } - let(:envelope) { OStatus2::Salmon.new.pack(payload, sender.keypair) } - let(:payload) { - <<~XML - - - carol@localdomain.com - carol - https://webdomain.com/users/carol - - - #{remote_status.id} - http://activitystrea.ms/schema/1.0/delete - - XML - } - - before do - receiver.update(locked: true) - remote_sender.update(private_key: sender.private_key, public_key: remote_sender.public_key) - end - - it 'deletes a record' do - expect(RemovalWorker).to receive(:perform_async).with(remote_status.id) - subject.call(envelope, receiver) - end - end - - describe 'follow request slap' do - before do - receiver.update(locked: true) - - payload = < - - bob - https://cb6e6126.ngrok.io/users/bob - - - someIdHere - http://activitystrea.ms/schema/1.0/request-friend - -XML - - envelope = OStatus2::Salmon.new.pack(payload, sender.keypair) - subject.call(envelope, receiver) - end - - it 'creates a record' do - expect(FollowRequest.find_by(account: sender, target_account: receiver)).to_not be_nil - end - end - - describe 'follow request slap from known remote user identified by email' do - before do - receiver.update(locked: true) - # Copy already-generated key - remote_sender.update(private_key: sender.private_key, public_key: remote_sender.public_key) - - payload = < - - carol@localdomain.com - carol - https://webdomain.com/users/carol - - - someIdHere - http://activitystrea.ms/schema/1.0/request-friend - -XML - - envelope = OStatus2::Salmon.new.pack(payload, remote_sender.keypair) - subject.call(envelope, receiver) - end - - it 'creates a record' do - expect(FollowRequest.find_by(account: remote_sender, target_account: receiver)).to_not be_nil - end - end - - describe 'follow request authorization slap' do - before do - receiver.update(locked: true) - FollowRequest.create(account: sender, target_account: receiver) - - payload = < - - alice - https://cb6e6126.ngrok.io/users/alice - - - someIdHere - http://activitystrea.ms/schema/1.0/authorize - -XML - - envelope = OStatus2::Salmon.new.pack(payload, receiver.keypair) - subject.call(envelope, sender) - end - - it 'creates a follow relationship' do - expect(Follow.find_by(account: sender, target_account: receiver)).to_not be_nil - end - - it 'removes the follow request' do - expect(FollowRequest.find_by(account: sender, target_account: receiver)).to be_nil - end - end - - describe 'follow request rejection slap' do - before do - receiver.update(locked: true) - FollowRequest.create(account: sender, target_account: receiver) - - payload = < - - alice - https://cb6e6126.ngrok.io/users/alice - - - someIdHere - http://activitystrea.ms/schema/1.0/reject - -XML - - envelope = OStatus2::Salmon.new.pack(payload, receiver.keypair) - subject.call(envelope, sender) - end - - it 'does not create a follow relationship' do - expect(Follow.find_by(account: sender, target_account: receiver)).to be_nil - end - - it 'removes the follow request' do - expect(FollowRequest.find_by(account: sender, target_account: receiver)).to be_nil - end - end -end diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb index 8a6bb44ac..35a804f2b 100644 --- a/spec/services/process_mentions_service_spec.rb +++ b/spec/services/process_mentions_service_spec.rb @@ -18,10 +18,6 @@ RSpec.describe ProcessMentionsService, type: :service do it 'creates a mention' do expect(remote_user.mentions.where(status: status).count).to eq 1 end - - it 'posts to remote user\'s Salmon end point' do - expect(a_request(:post, remote_user.salmon_url)).to have_been_made.once - end end context 'OStatus with private toot' do diff --git a/spec/services/pubsubhubbub/subscribe_service_spec.rb b/spec/services/pubsubhubbub/subscribe_service_spec.rb deleted file mode 100644 index 01c956230..000000000 --- a/spec/services/pubsubhubbub/subscribe_service_spec.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe Pubsubhubbub::SubscribeService, type: :service do - describe '#call' do - subject { described_class.new } - let(:user_account) { Fabricate(:account) } - - context 'with a nil account' do - it 'returns the invalid topic status results' do - result = service_call(account: nil) - - expect(result).to eq invalid_topic_status - end - end - - context 'with an invalid callback url' do - it 'returns invalid callback status when callback is blank' do - result = service_call(callback: '') - - expect(result).to eq invalid_callback_status - end - it 'returns invalid callback status when callback is not a URI' do - result = service_call(callback: 'invalid-hostname') - - expect(result).to eq invalid_callback_status - end - end - - context 'with a blocked domain in the callback' do - it 'returns callback not allowed' do - Fabricate(:domain_block, domain: 'test.host', severity: :suspend) - result = service_call(callback: 'https://test.host/api') - - expect(result).to eq not_allowed_callback_status - end - end - - context 'with a valid account and callback' do - it 'returns success status and confirms subscription' do - allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil) - subscription = Fabricate(:subscription, account: user_account) - - result = service_call(callback: subscription.callback_url) - expect(result).to eq success_status - expect(Pubsubhubbub::ConfirmationWorker).to have_received(:perform_async).with(subscription.id, 'subscribe', 'asdf', 3600) - end - end - end - - def service_call(account: user_account, callback: 'https://callback.host', secret: 'asdf', lease_seconds: 3600) - subject.call(account, callback, secret, lease_seconds) - end - - def invalid_topic_status - ['Invalid topic URL', 422] - end - - def invalid_callback_status - ['Invalid callback URL', 422] - end - - def not_allowed_callback_status - ['Callback URL not allowed', 403] - end - - def success_status - ['', 202] - end -end diff --git a/spec/services/pubsubhubbub/unsubscribe_service_spec.rb b/spec/services/pubsubhubbub/unsubscribe_service_spec.rb deleted file mode 100644 index 7ed9fc5af..000000000 --- a/spec/services/pubsubhubbub/unsubscribe_service_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe Pubsubhubbub::UnsubscribeService, type: :service do - describe '#call' do - subject { described_class.new } - - context 'with a nil account' do - it 'returns an invalid topic status' do - result = subject.call(nil, 'callback.host') - - expect(result).to eq invalid_topic_status - end - end - - context 'with a valid account' do - let(:account) { Fabricate(:account) } - - it 'returns a valid topic status and does not run confirm when no subscription' do - allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil) - result = subject.call(account, 'callback.host') - - expect(result).to eq valid_topic_status - expect(Pubsubhubbub::ConfirmationWorker).not_to have_received(:perform_async) - end - - it 'returns a valid topic status and does run confirm when there is a subscription' do - subscription = Fabricate(:subscription, account: account, callback_url: 'callback.host') - allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil) - result = subject.call(account, 'callback.host') - - expect(result).to eq valid_topic_status - expect(Pubsubhubbub::ConfirmationWorker).to have_received(:perform_async).with(subscription.id, 'unsubscribe') - end - end - - def invalid_topic_status - ['Invalid topic URL', 422] - end - - def valid_topic_status - ['', 202] - end - end -end diff --git a/spec/services/reblog_service_spec.rb b/spec/services/reblog_service_spec.rb index 9d84c41d5..58fb46f0f 100644 --- a/spec/services/reblog_service_spec.rb +++ b/spec/services/reblog_service_spec.rb @@ -46,10 +46,6 @@ RSpec.describe ReblogService, type: :service do it 'creates a reblog' do expect(status.reblogs.count).to eq 1 end - - it 'sends a Salmon slap for a remote reblog' do - expect(a_request(:post, 'http://salmon.example.com')).to have_been_made - end end context 'ActivityPub' do diff --git a/spec/services/reject_follow_service_spec.rb b/spec/services/reject_follow_service_spec.rb index e5ac37ed9..1aec060db 100644 --- a/spec/services/reject_follow_service_spec.rb +++ b/spec/services/reject_follow_service_spec.rb @@ -38,13 +38,6 @@ RSpec.describe RejectFollowService, type: :service do it 'does not create follow relation' do expect(bob.following?(sender)).to be false end - - it 'sends a follow request rejection salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:reject]) - }).to have_been_made.once - end end describe 'remote ActivityPub' do diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb index 7bba83a60..48191d47c 100644 --- a/spec/services/remove_status_service_spec.rb +++ b/spec/services/remove_status_service_spec.rb @@ -32,23 +32,10 @@ RSpec.describe RemoveStatusService, type: :service do expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id) end - it 'sends PuSH update to PuSH subscribers' do - expect(a_request(:post, 'http://example.com/push').with { |req| - req.body.match(OStatus::TagManager::VERBS[:delete]) - }).to have_been_made - end - it 'sends delete activity to followers' do expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.twice end - it 'sends Salmon slap to previously mentioned users' do - expect(a_request(:post, "http://example.com/salmon").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:delete]) - }).to have_been_made.once - end - it 'sends delete activity to rebloggers' do expect(a_request(:post, 'http://example2.com/inbox')).to have_been_made end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 27a85af7c..7a64f4161 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -6,19 +6,13 @@ RSpec.describe ResolveAccountService, type: :service do before do stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) - stub_request(:get, "https://redirected.com/.well-known/host-meta").to_return(request_fixture('redirected.host-meta.txt')) stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt')) - stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:gargron@redirected.com").to_return(request_fixture('webfinger.txt')) - stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:hacker1@redirected.com").to_return(request_fixture('webfinger-hacker1.txt')) - stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:hacker2@redirected.com").to_return(request_fixture('webfinger-hacker2.txt')) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) - stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt')) stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:get, "https://localdomain.com/.well-known/host-meta").to_return(request_fixture('localdomain-hostmeta.txt')) - stub_request(:get, "https://localdomain.com/.well-known/webfinger?resource=acct:foo@localdomain.com").to_return(status: 404) - stub_request(:get, "https://webdomain.com/.well-known/webfinger?resource=acct:foo@localdomain.com").to_return(request_fixture('localdomain-webfinger.txt')) - stub_request(:get, "https://webdomain.com/users/foo.atom").to_return(request_fixture('localdomain-feed.txt')) + stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) + stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt')) + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt')) + stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt')) + stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404) end it 'raises error if no such user can be resolved via webfinger' do @@ -29,74 +23,7 @@ RSpec.describe ResolveAccountService, type: :service do expect(subject.call('catsrgr8@example.com')).to be_nil end - it 'prevents hijacking existing accounts' do - account = subject.call('hacker1@redirected.com') - expect(account.salmon_url).to_not eq 'https://hacker.com/main/salmon/user/7477' - end - - it 'prevents hijacking inexisting accounts' do - expect(subject.call('hacker2@redirected.com')).to be_nil - end - - context 'with an OStatus account' do - it 'returns an already existing remote account' do - old_account = Fabricate(:account, username: 'gargron', domain: 'quitter.no') - returned_account = subject.call('gargron@quitter.no') - - expect(old_account.id).to eq returned_account.id - end - - it 'returns a new remote account' do - account = subject.call('gargron@quitter.no') - - expect(account.username).to eq 'gargron' - expect(account.domain).to eq 'quitter.no' - expect(account.remote_url).to eq 'https://quitter.no/api/statuses/user_timeline/7477.atom' - end - - it 'follows a legitimate account redirection' do - account = subject.call('gargron@redirected.com') - - expect(account.username).to eq 'gargron' - expect(account.domain).to eq 'quitter.no' - expect(account.remote_url).to eq 'https://quitter.no/api/statuses/user_timeline/7477.atom' - end - - it 'returns a new remote account' do - account = subject.call('foo@localdomain.com') - - expect(account.username).to eq 'foo' - expect(account.domain).to eq 'localdomain.com' - expect(account.remote_url).to eq 'https://webdomain.com/users/foo.atom' - end - end - context 'with an ActivityPub account' do - before do - stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt')) - stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt')) - stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt')) - stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404) - end - - it 'fallback to OStatus if actor json could not be fetched' do - stub_request(:get, "https://ap.example.com/users/foo").to_return(status: 404) - - account = subject.call('foo@ap.example.com') - - expect(account.ostatus?).to eq true - expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom' - end - - it 'fallback to OStatus if actor json did not have inbox_url' do - stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-noinbox.txt')) - - account = subject.call('foo@ap.example.com') - - expect(account.ostatus?).to eq true - expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom' - end - it 'returns new remote account' do account = subject.call('foo@ap.example.com') @@ -124,13 +51,14 @@ RSpec.describe ResolveAccountService, type: :service do it 'processes one remote account at a time using locks' do wait_for_start = true fail_occurred = false - return_values = [] + return_values = Concurrent::Array.new threads = Array.new(5) do Thread.new do true while wait_for_start + begin - return_values << described_class.new.call('foo@localdomain.com') + return_values << described_class.new.call('foo@ap.example.com') rescue ActiveRecord::RecordNotUnique fail_occurred = true end diff --git a/spec/services/send_interaction_service_spec.rb b/spec/services/send_interaction_service_spec.rb deleted file mode 100644 index 710d8184c..000000000 --- a/spec/services/send_interaction_service_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'rails_helper' - -RSpec.describe SendInteractionService, type: :service do - subject { SendInteractionService.new } - - it 'sends an XML envelope to the Salmon end point of remote user' -end diff --git a/spec/services/subscribe_service_spec.rb b/spec/services/subscribe_service_spec.rb deleted file mode 100644 index 10bdb1ba8..000000000 --- a/spec/services/subscribe_service_spec.rb +++ /dev/null @@ -1,43 +0,0 @@ -require 'rails_helper' - -RSpec.describe SubscribeService, type: :service do - let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') } - subject { SubscribeService.new } - - it 'sends subscription request to PuSH hub' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 202) - subject.call(account) - expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once - end - - it 'generates and keeps PuSH secret on successful call' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 202) - subject.call(account) - expect(account.secret).to_not be_blank - end - - it 'fails silently if PuSH hub forbids subscription' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 403) - subject.call(account) - end - - it 'fails silently if PuSH hub is not found' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 404) - subject.call(account) - end - - it 'fails loudly if there is a network error' do - stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error) - expect { subject.call(account) }.to raise_error HTTP::Error - end - - it 'fails loudly if PuSH hub is unavailable' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 503) - expect { subject.call(account) }.to raise_error Mastodon::UnexpectedResponseError - end - - it 'fails loudly if rate limited' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 429) - expect { subject.call(account) }.to raise_error Mastodon::UnexpectedResponseError - end -end diff --git a/spec/services/unblock_service_spec.rb b/spec/services/unblock_service_spec.rb index 5835b912b..6350c6834 100644 --- a/spec/services/unblock_service_spec.rb +++ b/spec/services/unblock_service_spec.rb @@ -30,13 +30,6 @@ RSpec.describe UnblockService, type: :service do it 'destroys the blocking relation' do expect(sender.blocking?(bob)).to be false end - - it 'sends an unblock salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:unblock]) - }).to have_been_made.once - end end describe 'remote ActivityPub' do diff --git a/spec/services/unfollow_service_spec.rb b/spec/services/unfollow_service_spec.rb index 8a2881ab1..84b5dafbc 100644 --- a/spec/services/unfollow_service_spec.rb +++ b/spec/services/unfollow_service_spec.rb @@ -30,13 +30,6 @@ RSpec.describe UnfollowService, type: :service do it 'destroys the following relation' do expect(sender.following?(bob)).to be false end - - it 'sends an unfollow salmon slap' do - expect(a_request(:post, "http://salmon.example.com/").with { |req| - xml = OStatus2::Salmon.new.unpack(req.body) - xml.match(OStatus::TagManager::VERBS[:unfollow]) - }).to have_been_made.once - end end describe 'remote ActivityPub' do diff --git a/spec/services/unsubscribe_service_spec.rb b/spec/services/unsubscribe_service_spec.rb deleted file mode 100644 index 54d4b1b53..000000000 --- a/spec/services/unsubscribe_service_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -require 'rails_helper' - -RSpec.describe UnsubscribeService, type: :service do - let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') } - subject { UnsubscribeService.new } - - it 'removes the secret and resets expiration on account' do - stub_request(:post, 'http://hub.example.com/').to_return(status: 204) - subject.call(account) - account.reload - - expect(account.secret).to be_blank - expect(account.subscription_expires_at).to be_blank - end - - it 'logs error on subscription failure' do - logger = stub_logger - stub_request(:post, 'http://hub.example.com/').to_return(status: 404) - subject.call(account) - - expect(logger).to have_received(:debug).with(/unsubscribe for bob@example.com failed/) - end - - it 'logs error on connection failure' do - logger = stub_logger - stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error) - subject.call(account) - - expect(logger).to have_received(:debug).with(/unsubscribe for bob@example.com failed/) - end - - def stub_logger - double(debug: nil).tap do |logger| - allow(Rails).to receive(:logger).and_return(logger) - end - end -end diff --git a/spec/services/update_remote_profile_service_spec.rb b/spec/services/update_remote_profile_service_spec.rb deleted file mode 100644 index f3ea70b80..000000000 --- a/spec/services/update_remote_profile_service_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -require 'rails_helper' - -RSpec.describe UpdateRemoteProfileService, type: :service do - let(:xml) { File.read(Rails.root.join('spec', 'fixtures', 'push', 'feed.atom')) } - - subject { UpdateRemoteProfileService.new } - - before do - stub_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png').to_return(request_fixture('avatar.txt')) - end - - context 'with updated details' do - let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com') } - - before do - subject.call(xml, remote_account) - end - - it 'downloads new avatar' do - expect(a_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png')).to have_been_made - end - - it 'sets the avatar remote url' do - expect(remote_account.reload.avatar_remote_url).to eq 'https://quitter.no/avatar/7477-300-20160211190340.png' - end - - it 'sets display name' do - expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ' - end - - it 'sets note' do - expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes' - end - end - - context 'with unchanged details' do - let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com', display_name: 'ＤＩＧＩＴＡＬ ＣＡＴ', note: 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes', avatar_remote_url: 'https://quitter.no/avatar/7477-300-20160211190340.png') } - - before do - subject.call(xml, remote_account) - end - - it 'does not re-download avatar' do - expect(a_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png')).to have_been_made.once - end - - it 'sets the avatar remote url' do - expect(remote_account.reload.avatar_remote_url).to eq 'https://quitter.no/avatar/7477-300-20160211190340.png' - end - - it 'sets display name' do - expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ' - end - - it 'sets note' do - expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes' - end - end - - context 'with updated details from a domain set to reject media' do - let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com') } - let!(:domain_block) { Fabricate(:domain_block, domain: 'example.com', reject_media: true) } - - before do - subject.call(xml, remote_account) - end - - it 'does not the avatar remote url' do - expect(remote_account.reload.avatar_remote_url).to be_nil - end - - it 'sets display name' do - expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ' - end - - it 'sets note' do - expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes' - end - - it 'does not set store the avatar' do - expect(remote_account.reload.avatar_file_name).to be_nil - end - end -end diff --git a/spec/workers/after_remote_follow_request_worker_spec.rb b/spec/workers/after_remote_follow_request_worker_spec.rb deleted file mode 100644 index bd623cca5..000000000 --- a/spec/workers/after_remote_follow_request_worker_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe AfterRemoteFollowRequestWorker do - subject { described_class.new } - let(:follow_request) { Fabricate(:follow_request) } - describe 'perform' do - context 'when the follow_request does not exist' do - it 'catches a raise and returns true' do - allow(FollowService).to receive(:new) - result = subject.perform('aaa') - - expect(result).to eq(true) - expect(FollowService).not_to have_received(:new) - end - end - - context 'when the account cannot be updated' do - it 'returns nil and does not call service when account is nil' do - allow(FollowService).to receive(:new) - service = double(call: nil) - allow(FetchRemoteAccountService).to receive(:new).and_return(service) - - result = subject.perform(follow_request.id) - - expect(result).to be_nil - expect(FollowService).not_to have_received(:new) - end - - it 'returns nil and does not call service when account is locked' do - allow(FollowService).to receive(:new) - service = double(call: double(locked?: true)) - allow(FetchRemoteAccountService).to receive(:new).and_return(service) - - result = subject.perform(follow_request.id) - - expect(result).to be_nil - expect(FollowService).not_to have_received(:new) - end - end - - context 'when the account is updated' do - it 'calls the follow service and destroys the follow' do - follow_service = double(call: nil) - allow(FollowService).to receive(:new).and_return(follow_service) - account = Fabricate(:account, locked: false) - service = double(call: account) - allow(FetchRemoteAccountService).to receive(:new).and_return(service) - - result = subject.perform(follow_request.id) - - expect(result).to be_nil - expect(follow_service).to have_received(:call).with(follow_request.account, account.acct) - expect { follow_request.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - end -end diff --git a/spec/workers/after_remote_follow_worker_spec.rb b/spec/workers/after_remote_follow_worker_spec.rb deleted file mode 100644 index d93c469f9..000000000 --- a/spec/workers/after_remote_follow_worker_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe AfterRemoteFollowWorker do - subject { described_class.new } - let(:follow) { Fabricate(:follow) } - describe 'perform' do - context 'when the follow does not exist' do - it 'catches a raise and returns true' do - allow(FollowService).to receive(:new) - result = subject.perform('aaa') - - expect(result).to eq(true) - expect(FollowService).not_to have_received(:new) - end - end - - context 'when the account cannot be updated' do - it 'returns nil and does not call service when account is nil' do - allow(FollowService).to receive(:new) - service = double(call: nil) - allow(FetchRemoteAccountService).to receive(:new).and_return(service) - - result = subject.perform(follow.id) - - expect(result).to be_nil - expect(FollowService).not_to have_received(:new) - end - - it 'returns nil and does not call service when account is not locked' do - allow(FollowService).to receive(:new) - service = double(call: double(locked?: false)) - allow(FetchRemoteAccountService).to receive(:new).and_return(service) - - result = subject.perform(follow.id) - - expect(result).to be_nil - expect(FollowService).not_to have_received(:new) - end - end - - context 'when the account is updated' do - it 'calls the follow service and destroys the follow' do - follow_service = double(call: nil) - allow(FollowService).to receive(:new).and_return(follow_service) - account = Fabricate(:account, locked: true) - service = double(call: account) - allow(FetchRemoteAccountService).to receive(:new).and_return(service) - - result = subject.perform(follow.id) - - expect(result).to be_nil - expect(follow_service).to have_received(:call).with(follow.account, account.acct) - expect { follow.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - end -end diff --git a/spec/workers/pubsubhubbub/confirmation_worker_spec.rb b/spec/workers/pubsubhubbub/confirmation_worker_spec.rb deleted file mode 100644 index 1eecdd2b5..000000000 --- a/spec/workers/pubsubhubbub/confirmation_worker_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe Pubsubhubbub::ConfirmationWorker do - include RoutingHelper - - subject { described_class.new } - - let!(:alice) { Fabricate(:account, username: 'alice') } - let!(:subscription) { Fabricate(:subscription, account: alice, callback_url: 'http://example.com/api', confirmed: false, expires_at: 3.days.from_now, secret: nil) } - - describe 'perform' do - describe 'with subscribe mode' do - it 'confirms and updates subscription when challenge matches' do - stub_random_value - stub_request(:get, url_for_mode('subscribe')) - .with(headers: http_headers) - .to_return(status: 200, body: challenge_value, headers: {}) - - seconds = 10.days.seconds.to_i - subject.perform(subscription.id, 'subscribe', 'asdf', seconds) - - subscription.reload - expect(subscription.secret).to eq 'asdf' - expect(subscription.confirmed).to eq true - expect(subscription.expires_at).to be_within(5).of(10.days.from_now) - end - - it 'does not update subscription when challenge does not match' do - stub_random_value - stub_request(:get, url_for_mode('subscribe')) - .with(headers: http_headers) - .to_return(status: 200, body: 'wrong value', headers: {}) - - seconds = 10.days.seconds.to_i - subject.perform(subscription.id, 'subscribe', 'asdf', seconds) - - subscription.reload - expect(subscription.secret).to be_blank - expect(subscription.confirmed).to eq false - expect(subscription.expires_at).to be_within(5).of(3.days.from_now) - end - end - - describe 'with unsubscribe mode' do - it 'confirms and destroys subscription when challenge matches' do - stub_random_value - stub_request(:get, url_for_mode('unsubscribe')) - .with(headers: http_headers) - .to_return(status: 200, body: challenge_value, headers: {}) - - seconds = 10.days.seconds.to_i - subject.perform(subscription.id, 'unsubscribe', 'asdf', seconds) - - expect { subscription.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - - it 'does not destroy subscription when challenge does not match' do - stub_random_value - stub_request(:get, url_for_mode('unsubscribe')) - .with(headers: http_headers) - .to_return(status: 200, body: 'wrong value', headers: {}) - - seconds = 10.days.seconds.to_i - subject.perform(subscription.id, 'unsubscribe', 'asdf', seconds) - - expect { subscription.reload }.not_to raise_error - end - end - end - - def url_for_mode(mode) - "http://example.com/api?hub.challenge=#{challenge_value}&hub.lease_seconds=863999&hub.mode=#{mode}&hub.topic=https://#{Rails.configuration.x.local_domain}/users/alice.atom" - end - - def stub_random_value - allow(SecureRandom).to receive(:hex).and_return(challenge_value) - end - - def challenge_value - '1a2s3d4f' - end - - def http_headers - { 'Connection' => 'close', 'Host' => 'example.com' } - end -end diff --git a/spec/workers/pubsubhubbub/delivery_worker_spec.rb b/spec/workers/pubsubhubbub/delivery_worker_spec.rb deleted file mode 100644 index c0e0d5186..000000000 --- a/spec/workers/pubsubhubbub/delivery_worker_spec.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe Pubsubhubbub::DeliveryWorker do - include RoutingHelper - subject { described_class.new } - - let(:payload) { 'test' } - - describe 'perform' do - it 'raises when subscription does not exist' do - expect { subject.perform 123, payload }.to raise_error(ActiveRecord::RecordNotFound) - end - - it 'does not attempt to deliver when domain blocked' do - _domain_block = Fabricate(:domain_block, domain: 'example.com', severity: :suspend) - subscription = Fabricate(:subscription, callback_url: 'https://example.com/api', last_successful_delivery_at: 2.days.ago) - - subject.perform(subscription.id, payload) - - expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(2.days.ago) - end - - it 'raises when request fails' do - subscription = Fabricate(:subscription) - - stub_request_to_respond_with(subscription, 500) - expect { subject.perform(subscription.id, payload) }.to raise_error Mastodon::UnexpectedResponseError - end - - it 'updates subscriptions when delivery succeeds' do - subscription = Fabricate(:subscription) - - stub_request_to_respond_with(subscription, 200) - subject.perform(subscription.id, payload) - - expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(Time.now.utc) - end - - it 'updates subscription without a secret when delivery succeeds' do - subscription = Fabricate(:subscription, secret: nil) - - stub_request_to_respond_with(subscription, 200) - subject.perform(subscription.id, payload) - - expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(Time.now.utc) - end - - def stub_request_to_respond_with(subscription, code) - stub_request(:post, 'http://example.com/callback') - .with(body: payload, headers: expected_headers(subscription)) - .to_return(status: code, body: '', headers: {}) - end - - def expected_headers(subscription) - { - 'Connection' => 'close', - 'Content-Type' => 'application/atom+xml', - 'Host' => 'example.com', - 'Link' => "; rel=\"hub\", ; rel=\"self\"", - }.tap do |basic| - known_digest = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), subscription.secret.to_s, payload) - basic.merge('X-Hub-Signature' => "sha1=#{known_digest}") if subscription.secret? - end - end - end -end diff --git a/spec/workers/pubsubhubbub/distribution_worker_spec.rb b/spec/workers/pubsubhubbub/distribution_worker_spec.rb deleted file mode 100644 index 584485079..000000000 --- a/spec/workers/pubsubhubbub/distribution_worker_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -require 'rails_helper' - -describe Pubsubhubbub::DistributionWorker do - subject { Pubsubhubbub::DistributionWorker.new } - - let!(:alice) { Fabricate(:account, username: 'alice') } - let!(:bob) { Fabricate(:account, username: 'bob', domain: 'example2.com') } - let!(:anonymous_subscription) { Fabricate(:subscription, account: alice, callback_url: 'http://example1.com', confirmed: true, lease_seconds: 3600) } - let!(:subscription_with_follower) { Fabricate(:subscription, account: alice, callback_url: 'http://example2.com', confirmed: true, lease_seconds: 3600) } - - before do - bob.follow!(alice) - end - - describe 'with public status' do - let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :public) } - - it 'delivers payload to all subscriptions' do - allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk) - subject.perform(status.stream_entry.id) - expect(Pubsubhubbub::DeliveryWorker).to have_received(:push_bulk).with([anonymous_subscription.id, subscription_with_follower.id]) - end - end - - context 'when OStatus privacy is not used' do - describe 'with private status' do - let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :private) } - - it 'does not deliver anything' do - allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk) - subject.perform(status.stream_entry.id) - expect(Pubsubhubbub::DeliveryWorker).to_not have_received(:push_bulk) - end - end - - describe 'with direct status' do - let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :direct) } - - it 'does not deliver payload' do - allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk) - subject.perform(status.stream_entry.id) - expect(Pubsubhubbub::DeliveryWorker).to_not have_received(:push_bulk) - end - end - end -end diff --git a/spec/workers/scheduler/subscriptions_scheduler_spec.rb b/spec/workers/scheduler/subscriptions_scheduler_spec.rb deleted file mode 100644 index a7d1046de..000000000 --- a/spec/workers/scheduler/subscriptions_scheduler_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'rails_helper' - -describe Scheduler::SubscriptionsScheduler do - subject { Scheduler::SubscriptionsScheduler.new } - - let!(:expiring_account1) { Fabricate(:account, subscription_expires_at: 20.minutes.from_now, domain: 'example.com', followers_count: 1, hub_url: 'http://hub.example.com') } - let!(:expiring_account2) { Fabricate(:account, subscription_expires_at: 4.hours.from_now, domain: 'example.org', followers_count: 1, hub_url: 'http://hub.example.org') } - - before do - stub_request(:post, 'http://hub.example.com/').to_return(status: 202) - stub_request(:post, 'http://hub.example.org/').to_return(status: 202) - end - - it 're-subscribes for all expiring accounts' do - subject.perform - expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once - expect(a_request(:post, 'http://hub.example.org/')).to have_been_made.once - end -end -- cgit From 406b46395d6f79e87b286585f6b6867374d198c1 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 7 Jul 2019 03:37:01 +0200 Subject: Fix URLs appearing twice in errors of ActivityPub::DeliveryWorker (#11231) --- app/lib/request.rb | 2 +- app/workers/activitypub/delivery_worker.rb | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'app/workers') diff --git a/app/lib/request.rb b/app/lib/request.rb index 5f7075a3c..322457ad7 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -59,7 +59,7 @@ class Request begin response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers)) rescue => e - raise e.class, "#{e.message} on #{@url}", e.backtrace[0] + raise e.class, "#{e.message} on #{@url}", e.backtrace end begin diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 818fd8f5d..8b52b8e49 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -18,13 +18,15 @@ class ActivityPub::DeliveryWorker @source_account = Account.find(source_account_id) @inbox_url = inbox_url @host = Addressable::URI.parse(inbox_url).normalized_site + @performed = false perform_request - - failure_tracker.track_success! - rescue => e - failure_tracker.track_failure! - raise e.class, "Delivery failed for #{inbox_url}: #{e.message}", e.backtrace[0] + ensure + if @performed + failure_tracker.track_success! + else + failure_tracker.track_failure! + end end private @@ -40,6 +42,8 @@ class ActivityPub::DeliveryWorker request_pool.with(@host) do |http_client| build_request(http_client).perform do |response| raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + + @performed = true end end end -- cgit From 5d3feed191bcbe2769512119752b426108152fe9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 10 Jul 2019 18:59:28 +0200 Subject: Refactor fetching of remote resources (#11251) --- app/helpers/jsonld_helper.rb | 47 ++++++----- app/lib/request.rb | 2 +- .../activitypub/fetch_remote_status_service.rb | 20 ++--- app/services/fetch_atom_service.rb | 93 --------------------- app/services/fetch_link_card_service.rb | 2 +- app/services/fetch_remote_account_service.rb | 2 +- app/services/fetch_remote_status_service.rb | 2 +- app/services/fetch_resource_service.rb | 68 +++++++++++++++ app/services/resolve_url_service.rb | 47 ++++------- app/workers/activitypub/delivery_worker.rb | 16 ++-- spec/services/fetch_atom_service_spec.rb | 96 ---------------------- spec/services/fetch_remote_account_service_spec.rb | 1 + spec/services/fetch_resource_service_spec.rb | 96 ++++++++++++++++++++++ spec/services/resolve_url_service_spec.rb | 44 ++-------- 14 files changed, 231 insertions(+), 305 deletions(-) delete mode 100644 app/services/fetch_atom_service.rb create mode 100644 app/services/fetch_resource_service.rb delete mode 100644 spec/services/fetch_atom_service_spec.rb create mode 100644 spec/services/fetch_resource_service_spec.rb (limited to 'app/workers') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 5b4011275..34a657e06 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -16,13 +16,15 @@ module JsonLdHelper # The url attribute can be a string, an array of strings, or an array of objects. # The objects could include a mimeType. Not-included mimeType means it's text/html. def url_to_href(value, preferred_type = nil) - single_value = if value.is_a?(Array) && !value.first.is_a?(String) - value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) } - elsif value.is_a?(Array) - value.first - else - value - end + single_value = begin + if value.is_a?(Array) && !value.first.is_a?(String) + value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) } + elsif value.is_a?(Array) + value.first + else + value + end + end if single_value.nil? || single_value.is_a?(String) single_value @@ -64,7 +66,9 @@ module JsonLdHelper def fetch_resource(uri, id, on_behalf_of = nil) unless id json = fetch_resource_without_id_validation(uri, on_behalf_of) + return unless json + uri = json['id'] end @@ -74,24 +78,26 @@ module JsonLdHelper def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false) build_request(uri, on_behalf_of).perform do |response| - unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - raise Mastodon::UnexpectedResponseError, response - end + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error + return body_to_json(response.body_with_limit) if response.code == 200 end + # If request failed, retry without doing it on behalf of a user return if on_behalf_of.nil? + build_request(uri).perform do |response| - unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - raise Mastodon::UnexpectedResponseError, response - end + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error + response.code == 200 ? body_to_json(response.body_with_limit) : nil end end def body_to_json(body, compare_id: nil) json = body.is_a?(String) ? Oj.load(body, mode: :strict) : body + return if compare_id.present? && json['id'] != compare_id + json rescue Oj::ParseError nil @@ -105,35 +111,34 @@ module JsonLdHelper end end - private - def response_successful?(response) (200...300).cover?(response.code) end def response_error_unsalvageable?(response) - (400...500).cover?(response.code) && response.code != 429 + response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) end def build_request(uri, on_behalf_of = nil) - request = Request.new(:get, uri) - request.on_behalf_of(on_behalf_of) if on_behalf_of - request.add_headers('Accept' => 'application/activity+json, application/ld+json') - request + Request.new(:get, uri).tap do |request| + request.on_behalf_of(on_behalf_of) if on_behalf_of + request.add_headers('Accept' => 'application/activity+json, application/ld+json') + end end def load_jsonld_context(url, _options = {}, &_block) json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do request = Request.new(:get, url) request.add_headers('Accept' => 'application/ld+json') - request.perform do |res| raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json' + res.body_with_limit end end doc = JSON::LD::API::RemoteDocument.new(url, json) + block_given? ? yield(doc) : doc end end diff --git a/app/lib/request.rb b/app/lib/request.rb index 322457ad7..1fd3f5190 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -41,7 +41,7 @@ class Request end def on_behalf_of(account, key_id_format = :acct, sign_with: nil) - raise ArgumentError unless account.local? + raise ArgumentError, 'account must be local' unless account&.local? @account = account @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 469821032..cf4f62899 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -5,18 +5,18 @@ class ActivityPub::FetchRemoteStatusService < BaseService # Should be called when uri has already been checked for locality def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil) - @json = if prefetched_body.nil? - fetch_resource(uri, id, on_behalf_of) - else - body_to_json(prefetched_body, compare_id: id ? uri : nil) - end + @json = begin + if prefetched_body.nil? + fetch_resource(uri, id, on_behalf_of) + else + body_to_json(prefetched_body, compare_id: id ? uri : nil) + end + end - return unless supported_context? && expected_type? - - return if actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id) + return if !(supported_context? && expected_type?) || actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id) actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account) - actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update(actor) + actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update?(actor) return if actor.nil? || actor.suspended? @@ -46,7 +46,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end - def needs_update(actor) + def needs_update?(actor) actor.possibly_stale? end end diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb deleted file mode 100644 index d6508a988..000000000 --- a/app/services/fetch_atom_service.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -class FetchAtomService < BaseService - include JsonLdHelper - - def call(url) - return if url.blank? - - result = process(url) - - # retry without ActivityPub - result ||= process(url) if @unsupported_activity - - result - rescue OpenSSL::SSL::SSLError => e - Rails.logger.debug "SSL error: #{e}" - nil - rescue HTTP::ConnectionError => e - Rails.logger.debug "HTTP ConnectionError: #{e}" - nil - end - - private - - def process(url, terminal = false) - @url = url - perform_request { |response| process_response(response, terminal) } - end - - def perform_request(&block) - accept = 'text/html' - accept = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/atom+xml, ' + accept unless @unsupported_activity - - Request.new(:get, @url).add_headers('Accept' => accept).perform(&block) - end - - def process_response(response, terminal = false) - return nil if response.code != 200 - - if response.mime_type == 'application/atom+xml' - [@url, { prefetched_body: response.body_with_limit }, :ostatus] - elsif ['application/activity+json', 'application/ld+json'].include?(response.mime_type) - body = response.body_with_limit - json = body_to_json(body) - if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present? - [json['id'], { prefetched_body: body, id: true }, :activitypub] - elsif supported_context?(json) && expected_type?(json) - [json['id'], { prefetched_body: body, id: true }, :activitypub] - else - @unsupported_activity = true - nil - end - elsif !terminal - link_header = response['Link'] && parse_link_header(response) - - if link_header&.find_link(%w(rel alternate)) - process_link_headers(link_header) - elsif response.mime_type == 'text/html' - process_html(response) - end - end - end - - def expected_type?(json) - equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) - end - - def process_html(response) - page = Nokogiri::HTML(response.body_with_limit) - - json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } - atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } - - result ||= process(json_link['href'], terminal: true) unless json_link.nil? || @unsupported_activity - result ||= process(atom_link['href'], terminal: true) unless atom_link.nil? - - result - end - - def process_link_headers(link_header) - json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) - atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml)) - - result ||= process(json_link.href, terminal: true) unless json_link.nil? || @unsupported_activity - result ||= process(atom_link.href, terminal: true) unless atom_link.nil? - - result - end - - def parse_link_header(response) - LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link']) - end -end diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 75fbd0e8c..4e75c370f 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -29,7 +29,7 @@ class FetchLinkCardService < BaseService end attach_card if @card&.persisted? - rescue HTTP::Error, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e Rails.logger.debug "Error fetching link #{@url}: #{e}" nil end diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb index a7f95603d..3cd06e30f 100644 --- a/app/services/fetch_remote_account_service.rb +++ b/app/services/fetch_remote_account_service.rb @@ -3,7 +3,7 @@ class FetchRemoteAccountService < BaseService def call(url, prefetched_body = nil, protocol = :ostatus) if prefetched_body.nil? - resource_url, resource_options, protocol = FetchAtomService.new.call(url) + resource_url, resource_options, protocol = FetchResourceService.new.call(url) else resource_url = url resource_options = { prefetched_body: prefetched_body } diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb index aac39dfd5..208dc7809 100644 --- a/app/services/fetch_remote_status_service.rb +++ b/app/services/fetch_remote_status_service.rb @@ -3,7 +3,7 @@ class FetchRemoteStatusService < BaseService def call(url, prefetched_body = nil, protocol = :ostatus) if prefetched_body.nil? - resource_url, resource_options, protocol = FetchAtomService.new.call(url) + resource_url, resource_options, protocol = FetchResourceService.new.call(url) else resource_url = url resource_options = { prefetched_body: prefetched_body } diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb new file mode 100644 index 000000000..c0473f3ad --- /dev/null +++ b/app/services/fetch_resource_service.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +class FetchResourceService < BaseService + include JsonLdHelper + + ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html' + + def call(url) + return if url.blank? + + process(url) + rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + Rails.logger.debug "Error fetching resource #{@url}: #{e}" + nil + end + + private + + def process(url, terminal = false) + @url = url + + perform_request { |response| process_response(response, terminal) } + end + + def perform_request(&block) + Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).perform(&block) + end + + def process_response(response, terminal = false) + return nil if response.code != 200 + + if ['application/activity+json', 'application/ld+json'].include?(response.mime_type) + body = response.body_with_limit + json = body_to_json(body) + + [json['id'], { prefetched_body: body, id: true }, :activitypub] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) || expected_type?(json)) + elsif !terminal + link_header = response['Link'] && parse_link_header(response) + + if link_header&.find_link(%w(rel alternate)) + process_link_headers(link_header) + elsif response.mime_type == 'text/html' + process_html(response) + end + end + end + + def expected_type?(json) + equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) + end + + def process_html(response) + page = Nokogiri::HTML(response.body_with_limit) + json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } + + process(json_link['href'], terminal: true) unless json_link.nil? + end + + def process_link_headers(link_header) + json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) + + process(json_link.href, terminal: true) unless json_link.nil? + end + + def parse_link_header(response) + LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link']) + end +end diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index f941b489a..80381c16b 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -4,64 +4,49 @@ class ResolveURLService < BaseService include JsonLdHelper include Authorization - attr_reader :url - def call(url, on_behalf_of: nil) - @url = url + @url = url @on_behalf_of = on_behalf_of - return process_local_url if local_url? - - process_url unless fetched_atom_feed.nil? + if local_url? + process_local_url + elsif !fetched_resource.nil? + process_url + end end private def process_url if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) - FetchRemoteAccountService.new.call(atom_url, body, protocol) + FetchRemoteAccountService.new.call(resource_url, body, protocol) elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) - FetchRemoteStatusService.new.call(atom_url, body, protocol) + FetchRemoteStatusService.new.call(resource_url, body, protocol) end end - def fetched_atom_feed - @_fetched_atom_feed ||= FetchAtomService.new.call(url) + def fetched_resource + @fetched_resource ||= FetchResourceService.new.call(@url) end - def atom_url - fetched_atom_feed.first + def resource_url + fetched_resource.first end def body - fetched_atom_feed.second[:prefetched_body] + fetched_resource.second[:prefetched_body] end def protocol - fetched_atom_feed.third + fetched_resource.third end def type return json_data['type'] if protocol == :activitypub - - case xml_root - when 'feed' - 'Person' - when 'entry' - 'Note' - end end def json_data - @_json_data ||= body_to_json(body) - end - - def xml_root - xml_data.root.name - end - - def xml_data - @_xml_data ||= Nokogiri::XML(body, nil, 'utf-8') + @json_data ||= body_to_json(body) end def local_url? @@ -83,10 +68,10 @@ class ResolveURLService < BaseService def check_local_status(status) return if status.nil? + authorize_with @on_behalf_of, status, :show? status rescue Mastodon::NotPermittedError - # Do not disclose the existence of status the user is not authorized to see nil end end diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 8b52b8e49..5457d9d4b 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -2,6 +2,7 @@ class ActivityPub::DeliveryWorker include Sidekiq::Worker + include JsonLdHelper STOPLIGHT_FAILURE_THRESHOLD = 10 STOPLIGHT_COOLDOWN = 60 @@ -32,9 +33,10 @@ class ActivityPub::DeliveryWorker private def build_request(http_client) - request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) - request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) - request.add_headers(HEADERS) + Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request| + request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) + request.add_headers(HEADERS) + end end def perform_request @@ -53,14 +55,6 @@ class ActivityPub::DeliveryWorker .run end - def response_successful?(response) - (200...300).cover?(response.code) - end - - def response_error_unsalvageable?(response) - response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) - end - def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end diff --git a/spec/services/fetch_atom_service_spec.rb b/spec/services/fetch_atom_service_spec.rb deleted file mode 100644 index 495540004..000000000 --- a/spec/services/fetch_atom_service_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -require 'rails_helper' - -RSpec.describe FetchAtomService, type: :service do - describe '#call' do - let(:url) { 'http://example.com' } - subject { FetchAtomService.new.call(url) } - - context 'url is blank' do - let(:url) { '' } - it { is_expected.to be_nil } - end - - context 'request failed' do - before do - WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {}) - end - - it { is_expected.to be_nil } - end - - context 'raise OpenSSL::SSL::SSLError' do - before do - allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError) - end - - it 'output log and return nil' do - expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('SSL error: OpenSSL::SSL::SSLError') - is_expected.to be_nil - end - end - - context 'raise HTTP::ConnectionError' do - before do - allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError) - end - - it 'output log and return nil' do - expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('HTTP ConnectionError: HTTP::ConnectionError') - is_expected.to be_nil - end - end - - context 'response success' do - let(:body) { '' } - let(:headers) { { 'Content-Type' => content_type } } - let(:json) { - { id: 1, - '@context': ActivityPub::TagManager::CONTEXT, - type: 'Note', - }.to_json - } - - before do - WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) - end - - context 'content type is application/atom+xml' do - let(:content_type) { 'application/atom+xml' } - - it { is_expected.to eq [url, { :prefetched_body => "" }, :ostatus] } - end - - context 'content_type is activity+json' do - let(:content_type) { 'application/activity+json; charset=utf-8' } - let(:body) { json } - - it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } - end - - context 'content_type is ld+json with profile' do - let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } - let(:body) { json } - - it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } - end - - before do - WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) - WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' }) - end - - context 'has link header' do - let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } - - it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } - end - - context 'content type is text/html' do - let(:content_type) { 'text/html' } - let(:body) { '' } - - it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } - end - end - end -end diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb index 37e9910d4..ee7325be2 100644 --- a/spec/services/fetch_remote_account_service_spec.rb +++ b/spec/services/fetch_remote_account_service_spec.rb @@ -4,6 +4,7 @@ RSpec.describe FetchRemoteAccountService, type: :service do let(:url) { 'https://example.com/alice' } let(:prefetched_body) { nil } let(:protocol) { :ostatus } + subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) } let(:actor) do diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb new file mode 100644 index 000000000..17c192c44 --- /dev/null +++ b/spec/services/fetch_resource_service_spec.rb @@ -0,0 +1,96 @@ +require 'rails_helper' + +RSpec.describe FetchResourceService, type: :service do + let!(:representative) { Fabricate(:account) } + + describe '#call' do + let(:url) { 'http://example.com' } + subject { described_class.new.call(url) } + + context 'url is blank' do + let(:url) { '' } + it { is_expected.to be_nil } + end + + context 'request failed' do + before do + WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {}) + end + + it { is_expected.to be_nil } + end + + context 'raise OpenSSL::SSL::SSLError' do + before do + allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError) + end + + it 'return nil' do + is_expected.to be_nil + end + end + + context 'raise HTTP::ConnectionError' do + before do + allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError) + end + + it 'return nil' do + is_expected.to be_nil + end + end + + context 'response success' do + let(:body) { '' } + let(:headers) { { 'Content-Type' => content_type } } + let(:json) { + { id: 1, + '@context': ActivityPub::TagManager::CONTEXT, + type: 'Note', + }.to_json + } + + before do + WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + end + + context 'content type is application/atom+xml' do + let(:content_type) { 'application/atom+xml' } + + it { is_expected.to eq nil } + end + + context 'content_type is activity+json' do + let(:content_type) { 'application/activity+json; charset=utf-8' } + let(:body) { json } + + it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } + end + + context 'content_type is ld+json with profile' do + let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } + let(:body) { json } + + it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } + end + + before do + WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' }) + end + + context 'has link header' do + let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } + + it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } + end + + context 'content type is text/html' do + let(:content_type) { 'text/html' } + let(:body) { '' } + + it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } + end + end + end +end diff --git a/spec/services/resolve_url_service_spec.rb b/spec/services/resolve_url_service_spec.rb index 7bb5d1940..aa4204637 100644 --- a/spec/services/resolve_url_service_spec.rb +++ b/spec/services/resolve_url_service_spec.rb @@ -6,48 +6,14 @@ describe ResolveURLService, type: :service do subject { described_class.new } describe '#call' do - it 'returns nil when there is no atom url' do - url = 'http://example.com/missing-atom' + it 'returns nil when there is no resource url' do + url = 'http://example.com/missing-resource' service = double - allow(FetchAtomService).to receive(:new).and_return service - allow(service).to receive(:call).with(url).and_return(nil) - - result = subject.call(url) - expect(result).to be_nil - end - - it 'fetches remote accounts for feed types' do - url = 'http://example.com/atom-feed' - service = double - allow(FetchAtomService).to receive(:new).and_return service - feed_url = 'http://feed-url' - feed_content = 'contents' - allow(service).to receive(:call).with(url).and_return([feed_url, { prefetched_body: feed_content }]) - - account_service = double - allow(FetchRemoteAccountService).to receive(:new).and_return(account_service) - allow(account_service).to receive(:call) - - _result = subject.call(url) - expect(account_service).to have_received(:call).with(feed_url, feed_content, nil) - end - - it 'fetches remote statuses for entry types' do - url = 'http://example.com/atom-entry' - service = double - allow(FetchAtomService).to receive(:new).and_return service - feed_url = 'http://feed-url' - feed_content = 'contents' - allow(service).to receive(:call).with(url).and_return([feed_url, { prefetched_body: feed_content }]) - - account_service = double - allow(FetchRemoteStatusService).to receive(:new).and_return(account_service) - allow(account_service).to receive(:call) - - _result = subject.call(url) + allow(FetchResourceService).to receive(:new).and_return service + allow(service).to receive(:call).with(url).and_return(nil) - expect(account_service).to have_received(:call).with(feed_url, feed_content, nil) + expect(subject.call(url)).to be_nil end end end -- cgit From 402302776c82c3853e723fe0c0c4dc99c69da3d9 Mon Sep 17 00:00:00 2001 From: "han@highemelry" Date: Sat, 13 Jul 2019 01:46:21 +0900 Subject: Change the retry limit in error of web push notification (#11292) - Change the maximum count of retry for web push notification (Default -> 5). - In case of high load of subscribe server, the retries will be repeated many times. - Because the retries occupy the default queue, maximum retry count should be reduced. --- app/workers/web/push_notification_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/web/push_notification_worker.rb b/app/workers/web/push_notification_worker.rb index 8e8a35973..901043975 100644 --- a/app/workers/web/push_notification_worker.rb +++ b/app/workers/web/push_notification_worker.rb @@ -3,7 +3,7 @@ class Web::PushNotificationWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options backtrace: true, retry: 5 def perform(subscription_id, notification_id) subscription = ::Web::PushSubscription.find(subscription_id) -- cgit From 0442fcdbc88c2a77b1216c38d2fb5fe11bec334d Mon Sep 17 00:00:00 2001 From: "han@highemelry" Date: Sat, 13 Jul 2019 01:46:21 +0900 Subject: Change the retry limit in error of web push notification (#11292) - Change the maximum count of retry for web push notification (Default -> 5). - In case of high load of subscribe server, the retries will be repeated many times. - Because the retries occupy the default queue, maximum retry count should be reduced. --- app/workers/web/push_notification_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/web/push_notification_worker.rb b/app/workers/web/push_notification_worker.rb index 8e8a35973..901043975 100644 --- a/app/workers/web/push_notification_worker.rb +++ b/app/workers/web/push_notification_worker.rb @@ -3,7 +3,7 @@ class Web::PushNotificationWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options backtrace: true, retry: 5 def perform(subscription_id, notification_id) subscription = ::Web::PushSubscription.find(subscription_id) -- cgit From b3f44aa186487b4f93f3a457607870c90caaf5df Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 15 Jul 2019 07:50:14 +0200 Subject: Add periodic removal of older thumbnails for preview cards (#11304) --- app/workers/maintenance/uncache_preview_worker.rb | 18 ++++++++++++++++++ .../scheduler/preview_cards_cleanup_scheduler.rb | 22 ++++++++++++++++++++++ config/sidekiq.yml | 3 +++ 3 files changed, 43 insertions(+) create mode 100644 app/workers/maintenance/uncache_preview_worker.rb create mode 100644 app/workers/scheduler/preview_cards_cleanup_scheduler.rb (limited to 'app/workers') diff --git a/app/workers/maintenance/uncache_preview_worker.rb b/app/workers/maintenance/uncache_preview_worker.rb new file mode 100644 index 000000000..810ffd8cc --- /dev/null +++ b/app/workers/maintenance/uncache_preview_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class Maintenance::UncachePreviewWorker + include Sidekiq::Worker + + sidekiq_options queue: 'pull' + + def perform(preview_card_id) + preview_card = PreviewCard.find(preview_card_id) + + return if preview_card.image.blank? + + preview_card.image.destroy + preview_card.save + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb new file mode 100644 index 000000000..2b38792f0 --- /dev/null +++ b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class Scheduler::PreviewCardsCleanupScheduler + include Sidekiq::Worker + + sidekiq_options unique: :until_executed, retry: 0 + + def perform + Maintenance::UncachePreviewWorker.push_bulk(recent_link_preview_cards.pluck(:id)) + Maintenance::UncachePreviewWorker.push_bulk(older_preview_cards.pluck(:id)) + end + + private + + def recent_link_preview_cards + PreviewCard.where(type: :link).where('updated_at < ?', 1.month.ago) + end + + def older_preview_cards + PreviewCard.where('updated_at < ?', 6.months.ago) + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index a16dea967..5c652792c 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -27,6 +27,9 @@ ip_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler + preview_cards_cleanup_scheduler: + cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' + class: Scheduler::PreviewCardsCleanupScheduler email_scheduler: cron: '0 10 * * 2' class: Scheduler::EmailScheduler -- cgit From bd87e6667975bc3b5bfaf3e1cdff97e041ed4c98 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 21 Jul 2019 04:08:00 +0200 Subject: Remove WebSub subscriptions (#11303) --- app/models/concerns/account_associations.rb | 3 - app/models/subscription.rb | 62 -------------------- app/policies/subscription_policy.rb | 7 --- app/services/suspend_account_service.rb | 1 - .../scheduler/subscriptions_cleanup_scheduler.rb | 4 +- config/sidekiq.yml | 3 - .../20190715031050_drop_subscriptions.rb | 11 ++++ db/schema.rb | 14 ----- spec/fabricators/subscription_fabricator.rb | 7 --- spec/models/subscription_spec.rb | 67 ---------------------- spec/policies/subscription_policy_spec.rb | 24 -------- .../services/batched_remove_status_service_spec.rb | 3 - spec/services/remove_status_service_spec.rb | 3 - spec/services/suspend_account_service_spec.rb | 8 +-- 14 files changed, 14 insertions(+), 203 deletions(-) delete mode 100644 app/models/subscription.rb delete mode 100644 app/policies/subscription_policy.rb create mode 100644 db/post_migrate/20190715031050_drop_subscriptions.rb delete mode 100644 spec/fabricators/subscription_fabricator.rb delete mode 100644 spec/models/subscription_spec.rb delete mode 100644 spec/policies/subscription_policy_spec.rb (limited to 'app/workers') diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index 0921e3252..1db7771c7 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -30,9 +30,6 @@ module AccountAssociations has_many :media_attachments, dependent: :destroy has_many :polls, dependent: :destroy - # PuSH subscriptions - has_many :subscriptions, dependent: :destroy - # Report relationships has_many :reports, dependent: :destroy, inverse_of: :account has_many :targeted_reports, class_name: 'Report', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account diff --git a/app/models/subscription.rb b/app/models/subscription.rb deleted file mode 100644 index 79b81828d..000000000 --- a/app/models/subscription.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true -# == Schema Information -# -# Table name: subscriptions -# -# id :bigint(8) not null, primary key -# callback_url :string default(""), not null -# secret :string -# expires_at :datetime -# confirmed :boolean default(FALSE), not null -# created_at :datetime not null -# updated_at :datetime not null -# last_successful_delivery_at :datetime -# domain :string -# account_id :bigint(8) not null -# - -class Subscription < ApplicationRecord - MIN_EXPIRATION = 1.day.to_i - MAX_EXPIRATION = 30.days.to_i - - belongs_to :account - - validates :callback_url, presence: true - validates :callback_url, uniqueness: { scope: :account_id } - - scope :confirmed, -> { where(confirmed: true) } - scope :future_expiration, -> { where(arel_table[:expires_at].gt(Time.now.utc)) } - scope :expired, -> { where(arel_table[:expires_at].lt(Time.now.utc)) } - scope :active, -> { confirmed.future_expiration } - - def lease_seconds=(value) - self.expires_at = future_expiration(value) - end - - def lease_seconds - (expires_at - Time.now.utc).to_i - end - - def expired? - Time.now.utc > expires_at - end - - before_validation :set_min_expiration - - private - - def future_expiration(value) - Time.now.utc + future_offset(value).seconds - end - - def future_offset(seconds) - [ - [MIN_EXPIRATION, seconds.to_i].max, - MAX_EXPIRATION, - ].min - end - - def set_min_expiration - self.lease_seconds = 0 unless expires_at - end -end diff --git a/app/policies/subscription_policy.rb b/app/policies/subscription_policy.rb deleted file mode 100644 index ac9a8a6c4..000000000 --- a/app/policies/subscription_policy.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class SubscriptionPolicy < ApplicationPolicy - def index? - admin? - end -end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 0ebe0b562..00cffcdfc 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -24,7 +24,6 @@ class SuspendAccountService < BaseService report_notes scheduled_statuses status_pins - subscriptions ).freeze ASSOCIATIONS_ON_DESTROY = %w( diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb index 5fba120f6..75fe681a9 100644 --- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb +++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb @@ -5,7 +5,5 @@ class Scheduler::SubscriptionsCleanupScheduler sidekiq_options unique: :until_executed, retry: 0 - def perform - Subscription.expired.in_batches.delete_all - end + def perform; end end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 5c652792c..7f41b6607 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -21,9 +21,6 @@ user_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *' class: Scheduler::UserCleanupScheduler - subscriptions_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(1..3) %> * * 0' - class: Scheduler::SubscriptionsCleanupScheduler ip_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler diff --git a/db/post_migrate/20190715031050_drop_subscriptions.rb b/db/post_migrate/20190715031050_drop_subscriptions.rb new file mode 100644 index 000000000..3719afe4a --- /dev/null +++ b/db/post_migrate/20190715031050_drop_subscriptions.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class DropSubscriptions < ActiveRecord::Migration[5.2] + def up + drop_table :subscriptions + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/schema.rb b/db/schema.rb index a6a14827b..6319dd932 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -648,19 +648,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true end - create_table "subscriptions", force: :cascade do |t| - t.string "callback_url", default: "", null: false - t.string "secret" - t.datetime "expires_at" - t.boolean "confirmed", default: false, null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.datetime "last_successful_delivery_at" - t.string "domain" - t.bigint "account_id", null: false - t.index ["account_id", "callback_url"], name: "index_subscriptions_on_account_id_and_callback_url", unique: true - end - create_table "tags", force: :cascade do |t| t.string "name", default: "", null: false t.datetime "created_at", null: false @@ -821,7 +808,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do add_foreign_key "statuses", "statuses", column: "reblog_of_id", on_delete: :cascade add_foreign_key "statuses_tags", "statuses", on_delete: :cascade add_foreign_key "statuses_tags", "tags", name: "fk_3081861e21", on_delete: :cascade - add_foreign_key "subscriptions", "accounts", name: "fk_9847d1cbb5", on_delete: :cascade add_foreign_key "tombstones", "accounts", on_delete: :cascade add_foreign_key "user_invite_requests", "users", on_delete: :cascade add_foreign_key "users", "accounts", name: "fk_50500f500d", on_delete: :cascade diff --git a/spec/fabricators/subscription_fabricator.rb b/spec/fabricators/subscription_fabricator.rb deleted file mode 100644 index 347dab5df..000000000 --- a/spec/fabricators/subscription_fabricator.rb +++ /dev/null @@ -1,7 +0,0 @@ -Fabricator(:subscription) do - account - callback_url "http://example.com/callback" - secret "foobar" - expires_at "2016-11-28 11:30:07" - confirmed false -end diff --git a/spec/models/subscription_spec.rb b/spec/models/subscription_spec.rb deleted file mode 100644 index b83979d13..000000000 --- a/spec/models/subscription_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'rails_helper' - -RSpec.describe Subscription, type: :model do - let(:alice) { Fabricate(:account, username: 'alice') } - - subject { Fabricate(:subscription, account: alice) } - - describe '#expired?' do - it 'return true when expires_at is past' do - subject.expires_at = 2.days.ago - expect(subject.expired?).to be true - end - - it 'return false when expires_at is future' do - subject.expires_at = 2.days.from_now - expect(subject.expired?).to be false - end - end - - describe 'lease_seconds' do - it 'returns the time remaining until expiration' do - datetime = 1.day.from_now - subscription = Subscription.new(expires_at: datetime) - travel_to(datetime - 12.hours) do - expect(subscription.lease_seconds).to eq(12.hours) - end - end - end - - describe 'lease_seconds=' do - it 'sets expires_at to min expiration when small value is provided' do - subscription = Subscription.new - datetime = 1.day.from_now - too_low = Subscription::MIN_EXPIRATION - 1000 - travel_to(datetime) do - subscription.lease_seconds = too_low - end - - expected = datetime + Subscription::MIN_EXPIRATION.seconds - expect(subscription.expires_at).to be_within(1.0).of(expected) - end - - it 'sets expires_at to value when valid value is provided' do - subscription = Subscription.new - datetime = 1.day.from_now - valid = Subscription::MIN_EXPIRATION + 1000 - travel_to(datetime) do - subscription.lease_seconds = valid - end - - expected = datetime + valid.seconds - expect(subscription.expires_at).to be_within(1.0).of(expected) - end - - it 'sets expires_at to max expiration when large value is provided' do - subscription = Subscription.new - datetime = 1.day.from_now - too_high = Subscription::MAX_EXPIRATION + 1000 - travel_to(datetime) do - subscription.lease_seconds = too_high - end - - expected = datetime + Subscription::MAX_EXPIRATION.seconds - expect(subscription.expires_at).to be_within(1.0).of(expected) - end - end -end diff --git a/spec/policies/subscription_policy_spec.rb b/spec/policies/subscription_policy_spec.rb deleted file mode 100644 index 21d60c15f..000000000 --- a/spec/policies/subscription_policy_spec.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' -require 'pundit/rspec' - -RSpec.describe SubscriptionPolicy do - let(:subject) { described_class } - let(:admin) { Fabricate(:user, admin: true).account } - let(:john) { Fabricate(:user).account } - - permissions :index? do - context 'admin?' do - it 'permits' do - expect(subject).to permit(admin, Subscription) - end - end - - context '!admin?' do - it 'denies' do - expect(subject).to_not permit(john, Subscription) - end - end - end -end diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb index d52e7f484..f84256f18 100644 --- a/spec/services/batched_remove_status_service_spec.rb +++ b/spec/services/batched_remove_status_service_spec.rb @@ -14,11 +14,8 @@ RSpec.describe BatchedRemoveStatusService, type: :service do before do allow(Redis.current).to receive_messages(publish: nil) - stub_request(:post, 'http://example.com/push').to_return(status: 200, body: '', headers: {}) - stub_request(:post, 'http://example.com/salmon').to_return(status: 200, body: '', headers: {}) stub_request(:post, 'http://example.com/inbox').to_return(status: 200) - Fabricate(:subscription, account: alice, callback_url: 'http://example.com/push', confirmed: true, expires_at: 30.days.from_now) jeff.user.update(current_sign_in_at: Time.zone.now) jeff.follow!(alice) hank.follow!(alice) diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb index 48191d47c..06676ec45 100644 --- a/spec/services/remove_status_service_spec.rb +++ b/spec/services/remove_status_service_spec.rb @@ -10,12 +10,9 @@ RSpec.describe RemoveStatusService, type: :service do let!(:bill) { Fabricate(:account, username: 'bill', protocol: :activitypub, domain: 'example2.com', inbox_url: 'http://example2.com/inbox') } before do - stub_request(:post, 'http://example.com/push').to_return(status: 200, body: '', headers: {}) - stub_request(:post, 'http://example.com/salmon').to_return(status: 200, body: '', headers: {}) stub_request(:post, 'http://example.com/inbox').to_return(status: 200) stub_request(:post, 'http://example2.com/inbox').to_return(status: 200) - Fabricate(:subscription, account: alice, callback_url: 'http://example.com/push', confirmed: true, expires_at: 30.days.from_now) jeff.follow!(alice) hank.follow!(alice) diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb index 896ac17a3..eebbbc12a 100644 --- a/spec/services/suspend_account_service_spec.rb +++ b/spec/services/suspend_account_service_spec.rb @@ -18,7 +18,6 @@ RSpec.describe SuspendAccountService, type: :service do let!(:favourite) { Fabricate(:favourite, account: account) } let!(:active_relationship) { Fabricate(:follow, account: account) } let!(:passive_relationship) { Fabricate(:follow, target_account: account) } - let!(:subscription) { Fabricate(:subscription, account: account) } let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', protocol: :activitypub) } let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) } @@ -31,9 +30,8 @@ RSpec.describe SuspendAccountService, type: :service do account.favourites, account.active_relationships, account.passive_relationships, - account.subscriptions ].map(&:count) - }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0]) + }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0]) end it 'sends a delete actor activity to all known inboxes' do @@ -62,7 +60,6 @@ RSpec.describe SuspendAccountService, type: :service do let!(:favourite) { Fabricate(:favourite, account: remote_bob) } let!(:active_relationship) { Fabricate(:follow, account: remote_bob, target_account: account) } let!(:passive_relationship) { Fabricate(:follow, target_account: remote_bob) } - let!(:subscription) { Fabricate(:subscription, account: remote_bob) } it 'deletes associated records' do is_expected.to change { @@ -73,9 +70,8 @@ RSpec.describe SuspendAccountService, type: :service do remote_bob.favourites, remote_bob.active_relationships, remote_bob.passive_relationships, - remote_bob.subscriptions ].map(&:count) - }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0]) + }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0]) end it 'sends a reject follow to follwer inboxes' do -- cgit From 0d80f686d81e4a12db1ac54b8f31f69f375dc8dc Mon Sep 17 00:00:00 2001 From: mayaeh Date: Sun, 28 Jul 2019 20:48:19 +0900 Subject: Add `tootctl preview_cards remove` (#11320) * Add `tootctl preview_cards remove` * fix code style * Remove `Scheduler::PreviewCardsCleanupScheduler` file * fix code style again Add exclude case where image_file_name is blank * Added a function to output confirmation if the specified number of days is less than 2 weeks --- .../scheduler/preview_cards_cleanup_scheduler.rb | 22 ----- config/sidekiq.yml | 3 - lib/cli.rb | 4 + lib/mastodon/preview_cards_cli.rb | 94 ++++++++++++++++++++++ 4 files changed, 98 insertions(+), 25 deletions(-) delete mode 100644 app/workers/scheduler/preview_cards_cleanup_scheduler.rb create mode 100644 lib/mastodon/preview_cards_cli.rb (limited to 'app/workers') diff --git a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb deleted file mode 100644 index 2b38792f0..000000000 --- a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::PreviewCardsCleanupScheduler - include Sidekiq::Worker - - sidekiq_options unique: :until_executed, retry: 0 - - def perform - Maintenance::UncachePreviewWorker.push_bulk(recent_link_preview_cards.pluck(:id)) - Maintenance::UncachePreviewWorker.push_bulk(older_preview_cards.pluck(:id)) - end - - private - - def recent_link_preview_cards - PreviewCard.where(type: :link).where('updated_at < ?', 1.month.ago) - end - - def older_preview_cards - PreviewCard.where('updated_at < ?', 6.months.ago) - end -end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 7f41b6607..6ebe450b0 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -24,9 +24,6 @@ ip_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler - preview_cards_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' - class: Scheduler::PreviewCardsCleanupScheduler email_scheduler: cron: '0 10 * * 2' class: Scheduler::EmailScheduler diff --git a/lib/cli.rb b/lib/cli.rb index be276583d..fbdf49fc3 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -9,6 +9,7 @@ require_relative 'mastodon/search_cli' require_relative 'mastodon/settings_cli' require_relative 'mastodon/statuses_cli' require_relative 'mastodon/domains_cli' +require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/version' @@ -42,6 +43,9 @@ module Mastodon desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains' subcommand 'domains', Mastodon::DomainsCLI + desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards' + subcommand 'preview_cards', Mastodon::PreviewCardsCLI + desc 'cache SUBCOMMAND ...ARGS', 'Manage cache' subcommand 'cache', Mastodon::CacheCLI diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb new file mode 100644 index 000000000..465fe7d0b --- /dev/null +++ b/lib/mastodon/preview_cards_cli.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class PreviewCardsCLI < Thor + include ActionView::Helpers::NumberHelper + + def self.exit_on_failure? + true + end + + option :days, type: :numeric, default: 180 + option :background, type: :boolean, default: false + option :verbose, type: :boolean, default: false + option :dry_run, type: :boolean, default: false + option :link, type: :boolean, default: false + desc 'remove', 'Remove preview cards' + long_desc <<-DESC + Removes locally thumbnails for previews. + + The --days option specifies how old preview cards have to be before + they are removed. It defaults to 180 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + + With the --dry-run option, no work will be done. + + With the --verbose option, when preview cards are processed sequentially in the + foreground, the IDs of the preview cards will be printed. + + With the --link option, delete only link-type preview cards. + DESC + def remove + prompt = TTY::Prompt.new + time_ago = options[:days].days.ago + queued = 0 + processed = 0 + size = 0 + dry_run = options[:dry_run] ? '(DRY RUN)' : '' + link = options[:link] ? 'link-type ' : '' + scope = PreviewCard.where.not(image_file_name: nil) + scope = scope.where.not(image_file_name: '') + scope = scope.where(type: :link) if options[:link] + scope = scope.where('updated_at < ?', time_ago) + + if time_ago > 2.weeks.ago + prompt.say "\n" + prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.') + prompt.say "\n" + + unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false) + prompt.say "\n" + prompt.warn 'Nothing execute. Bye!' + prompt.say "\n" + exit(1) + end + end + + if options[:background] + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + queued += preview_cards.size + size += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) } + Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run] + end + + else + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + preview_cards.each do |p| + size += p.image_file_size || 0 + Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run] + options[:verbose] ? say(p.id) : say('.', :green, false) + processed += 1 + end + end + end + + say + + if options[:background] + say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + else + say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + end + end + end +end -- cgit From bced70469a6c4aecdb3c71055f329a0f579eb14c Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 7 Aug 2019 20:20:23 +0200 Subject: Add domain block notes (#11515) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add database columns for adding notes to domain blocks/restrctions * Add admin UI to set private and public comments when blocking a domain * Add text for private and public comments on domain blocks * Show domain block comments in admin UI * Add comments to the domain block undo page * Make UnblockDomainService more robust regarding upgraded domain blocks * Allow editing domain blocks * Rename button from “undo domain block” to “view domain block” in account admin UI * Change test to unsilence silenced users from upgraded blocks --- app/controllers/admin/domain_blocks_controller.rb | 28 ++++++++++++++++++-- app/controllers/admin/instances_controller.rb | 2 ++ app/models/domain_block.rb | 16 +++++++----- app/services/block_domain_service.rb | 11 +++++++- app/services/unblock_domain_service.rb | 19 ++------------ app/views/admin/accounts/show.html.haml | 2 +- app/views/admin/domain_blocks/edit.html.haml | 30 ++++++++++++++++++++++ app/views/admin/domain_blocks/new.html.haml | 6 +++++ app/views/admin/domain_blocks/show.html.haml | 12 +++++++++ app/views/admin/instances/show.html.haml | 13 ++++++++++ app/workers/domain_block_worker.rb | 4 +-- config/locales/en.yml | 8 ++++++ config/routes.rb | 6 ++++- ...20190807135426_add_comments_to_domain_blocks.rb | 7 +++++ db/schema.rb | 4 ++- spec/services/unblock_domain_service_spec.rb | 2 +- spec/workers/domain_block_worker_spec.rb | 2 +- 17 files changed, 138 insertions(+), 34 deletions(-) create mode 100644 app/views/admin/domain_blocks/edit.html.haml create mode 100644 db/migrate/20190807135426_add_comments_to_domain_blocks.rb (limited to 'app/workers') diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index 7129656da..74a36b79c 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -2,13 +2,17 @@ module Admin class DomainBlocksController < BaseController - before_action :set_domain_block, only: [:show, :destroy] + before_action :set_domain_block, only: [:show, :destroy, :edit, :update] def new authorize :domain_block, :create? @domain_block = DomainBlock.new(domain: params[:_domain]) end + def edit + authorize :domain_block, :create? + end + def create authorize :domain_block, :create? @@ -35,6 +39,22 @@ module Admin end end + def update + authorize :domain_block, :create? + + @domain_block.update(update_params) + + severity_changed = @domain_block.severity_changed? + + if @domain_block.save + DomainBlockWorker.perform_async(@domain_block.id, severity_changed) + log_action :create, @domain_block + redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg') + else + render :edit + end + end + def show authorize @domain_block, :show? end @@ -52,8 +72,12 @@ module Admin @domain_block = DomainBlock.find(params[:id]) end + def update_params + params.require(:domain_block).permit(:severity, :reject_media, :reject_reports, :private_comment, :public_comment) + end + def resource_params - params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports) + params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment) end end end diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb index d4f201807..b47b18f8e 100644 --- a/app/controllers/admin/instances_controller.rb +++ b/app/controllers/admin/instances_controller.rb @@ -21,6 +21,8 @@ module Admin @blocks_count = Block.where(target_account: Account.where(domain: params[:id])).count @available = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url) @media_storage = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size) + @private_comment = @domain_block&.private_comment + @public_comment = @domain_block&.public_comment end private diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index 25d3b87ef..3f5b9f23e 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -3,13 +3,15 @@ # # Table name: domain_blocks # -# id :bigint(8) not null, primary key -# domain :string default(""), not null -# created_at :datetime not null -# updated_at :datetime not null -# severity :integer default("silence") -# reject_media :boolean default(FALSE), not null -# reject_reports :boolean default(FALSE), not null +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# severity :integer default("silence") +# reject_media :boolean default(FALSE), not null +# reject_reports :boolean default(FALSE), not null +# private_comment :text +# public_comment :text # class DomainBlock < ApplicationRecord diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb index c5e5e5761..0ec6be503 100644 --- a/app/services/block_domain_service.rb +++ b/app/services/block_domain_service.rb @@ -3,13 +3,22 @@ class BlockDomainService < BaseService attr_reader :domain_block - def call(domain_block) + def call(domain_block, update = false) @domain_block = domain_block process_domain_block! + process_retroactive_updates! if update end private + def process_retroactive_updates! + # If the domain block severity has been changed, undo the appropriate limitations + scope = Account.by_domain_and_subdomains(domain_block.domain) + + scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.silence? + scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) unless domain_block.suspend? + end + def process_domain_block! clear_media! if domain_block.reject_media? diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb index fc262a50a..d502d9e49 100644 --- a/app/services/unblock_domain_service.rb +++ b/app/services/unblock_domain_service.rb @@ -10,24 +10,9 @@ class UnblockDomainService < BaseService end def process_retroactive_updates - blocked_accounts.in_batches.update_all(update_options) unless domain_block.noop? - end - - def blocked_accounts scope = Account.by_domain_and_subdomains(domain_block.domain) - if domain_block.silence? - scope.where(silenced_at: @domain_block.created_at) - else - scope.where(suspended_at: @domain_block.created_at) - end - end - - def update_options - { domain_block_impact => nil } - end - - def domain_block_impact - domain_block.silence? ? :silenced_at : :suspended_at + scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.noop? + scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) if domain_block.suspend? end end diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml index 7494c9fa2..59babd3b0 100644 --- a/app/views/admin/accounts/show.html.haml +++ b/app/views/admin/accounts/show.html.haml @@ -174,7 +174,7 @@ - unless @account.local? - if DomainBlock.where(domain: @account.domain).exists? - = link_to t('admin.domain_blocks.undo'), admin_instance_path(@account.domain), class: 'button' + = link_to t('admin.domain_blocks.view'), admin_instance_path(@account.domain), class: 'button' - else = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @account.domain), class: 'button button--destructive' diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml new file mode 100644 index 000000000..29e47ef3b --- /dev/null +++ b/app/views/admin/domain_blocks/edit.html.haml @@ -0,0 +1,30 @@ +- content_for :header_tags do + = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous' + +- content_for :page_title do + = t('admin.domain_blocks.edit') + += simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :put do |f| + = render 'shared/error_messages', object: @domain_block + + .fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), hint: t('admin.domain_blocks.new.hint'), required: true, readonly: true, disabled: true + + .fields-row__column.fields-row__column-6.fields-group + = f.input :severity, collection: DomainBlock.severities.keys, wrapper: :with_label, include_blank: false, label_method: lambda { |type| t("admin.domain_blocks.new.severity.#{type}") }, hint: t('admin.domain_blocks.new.severity.desc_html') + + .fields-group + = f.input :reject_media, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_media'), hint: I18n.t('admin.domain_blocks.reject_media_hint') + + .fields-group + = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint') + + .field-group + = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6 + + .field-group + = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6 + + .actions + = f.button :button, t('generic.save_changes'), type: :submit diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml index 055d2fbd7..ed1581936 100644 --- a/app/views/admin/domain_blocks/new.html.haml +++ b/app/views/admin/domain_blocks/new.html.haml @@ -20,5 +20,11 @@ .fields-group = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint') + .field-group + = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6 + + .field-group + = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6 + .actions = f.button :button, t('.create'), type: :submit diff --git a/app/views/admin/domain_blocks/show.html.haml b/app/views/admin/domain_blocks/show.html.haml index dca4dbac7..e64aaa629 100644 --- a/app/views/admin/domain_blocks/show.html.haml +++ b/app/views/admin/domain_blocks/show.html.haml @@ -1,6 +1,18 @@ - content_for :page_title do = t('admin.domain_blocks.show.title', domain: @domain_block.domain) +- if @domain_block.private_comment.present? + .speech-bubble + .speech-bubble__bubble + = simple_format(h(@domain_block.private_comment)) + .speech-bubble__owner= t 'admin.instances.private_comment' + +- if @domain_block.public_comment.present? + .speech-bubble + .speech-bubble__bubble + = simple_format(h(@domain_block.public_comment)) + .speech-bubble__owner= t 'admin.instances.public_comment' + = simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :delete do |f| - unless (@domain_block.noop?) diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml index fbb49ba02..294c9495d 100644 --- a/app/views/admin/instances/show.html.haml +++ b/app/views/admin/instances/show.html.haml @@ -31,6 +31,18 @@ = fa_icon 'times' .dashboard__counters__label= t 'admin.instances.delivery_available' +- if @private_comment.present? + .speech-bubble + .speech-bubble__bubble + = simple_format(h(@private_comment)) + .speech-bubble__owner= t 'admin.instances.private_comment' + +- if @public_comment.present? + .speech-bubble + .speech-bubble__bubble + = simple_format(h(@public_comment)) + .speech-bubble__owner= t 'admin.instances.public_comment' + %hr.spacer/ %div{ style: 'overflow: hidden' } @@ -41,6 +53,7 @@ - if @domain_allow = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete } - elsif @domain_block + = link_to t('admin.domain_blocks.edit'), edit_admin_domain_block_path(@domain_block), class: 'button' = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button' - else = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button' diff --git a/app/workers/domain_block_worker.rb b/app/workers/domain_block_worker.rb index 884477829..35518d6b5 100644 --- a/app/workers/domain_block_worker.rb +++ b/app/workers/domain_block_worker.rb @@ -3,8 +3,8 @@ class DomainBlockWorker include Sidekiq::Worker - def perform(domain_block_id) - BlockDomainService.new.call(DomainBlock.find(domain_block_id)) + def perform(domain_block_id, update = false) + BlockDomainService.new.call(DomainBlock.find(domain_block_id), update) rescue ActiveRecord::RecordNotFound true end diff --git a/config/locales/en.yml b/config/locales/en.yml index 17ff24726..b677a6651 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -284,6 +284,7 @@ en: created_msg: Domain block is now being processed destroyed_msg: Domain block has been undone domain: Domain + edit: Edit domain block existing_domain_block_html: You have already imposed stricter limits on %{name}, you need to unblock it first. new: create: Create block @@ -294,6 +295,10 @@ en: silence: Silence suspend: Suspend title: New domain block + private_comment: Private comment + private_comment_hint: Comment about this domain limitation for internal use by the moderators. + public_comment: Public comment + public_comment_hint: Comment about this domain limitation for the general public, if advertising the list of domain limitations is enabled. reject_media: Reject media files reject_media_hint: Removes locally stored media files and refuses to download any in the future. Irrelevant for suspensions reject_reports: Reject reports @@ -313,6 +318,7 @@ en: title: Undo domain block for %{domain} undo: Undo undo: Undo domain block + view: View domain block email_domain_blocks: add_new: Add new created_msg: Successfully added e-mail domain to blacklist @@ -336,6 +342,8 @@ en: all: All limited: Limited title: Moderation + private_comment: Private comment + public_comment: Public comment title: Federation total_blocked_by_us: Blocked by us total_followed_by_them: Followed by them diff --git a/config/routes.rb b/config/routes.rb index 60f7d2e05..9c33b8190 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -155,7 +155,11 @@ Rails.application.routes.draw do get '/dashboard', to: 'dashboard#index' resources :domain_allows, only: [:new, :create, :show, :destroy] - resources :domain_blocks, only: [:new, :create, :show, :destroy] + resources :domain_blocks, only: [:new, :create, :show, :destroy, :update] do + member do + get :edit + end + end resources :email_domain_blocks, only: [:index, :new, :create, :destroy] resources :action_logs, only: [:index] resources :warning_presets, except: [:new] diff --git a/db/migrate/20190807135426_add_comments_to_domain_blocks.rb b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb new file mode 100644 index 000000000..b660a71ad --- /dev/null +++ b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb @@ -0,0 +1,7 @@ +class AddCommentsToDomainBlocks < ActiveRecord::Migration[5.2] + def change + add_column :domain_blocks, :private_comment, :text + add_column :domain_blocks, :public_comment, :text + end +end + diff --git a/db/schema.rb b/db/schema.rb index d1b6825b4..f8fc6a821 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_05_123746) do +ActiveRecord::Schema.define(version: 2019_08_07_135426) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -259,6 +259,8 @@ ActiveRecord::Schema.define(version: 2019_08_05_123746) do t.integer "severity", default: 0 t.boolean "reject_media", default: false, null: false t.boolean "reject_reports", default: false, null: false + t.text "private_comment" + t.text "public_comment" t.index ["domain"], name: "index_domain_blocks_on_domain", unique: true end diff --git a/spec/services/unblock_domain_service_spec.rb b/spec/services/unblock_domain_service_spec.rb index 619aefb5c..27dbc92ad 100644 --- a/spec/services/unblock_domain_service_spec.rb +++ b/spec/services/unblock_domain_service_spec.rb @@ -31,7 +31,7 @@ describe UnblockDomainService, type: :service do subject.call(@domain_block) expect_deleted_domain_block expect(@suspended.reload.suspended?).to be false - expect(@silenced.reload.silenced?).to be true + expect(@silenced.reload.silenced?).to be false expect(@independently_suspended.reload.suspended?).to be true expect(@independently_silenced.reload.silenced?).to be true end diff --git a/spec/workers/domain_block_worker_spec.rb b/spec/workers/domain_block_worker_spec.rb index c4138501f..48b3e38c4 100644 --- a/spec/workers/domain_block_worker_spec.rb +++ b/spec/workers/domain_block_worker_spec.rb @@ -14,7 +14,7 @@ describe DomainBlockWorker do result = subject.perform(domain_block.id) expect(result).to be_nil - expect(service).to have_received(:call).with(domain_block) + expect(service).to have_received(:call).with(domain_block, false) end it 'calls domain block service for relevant domain block' do -- cgit From 97192d9a77c0b4b68afe50d6a94d87110a8adbcd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Aug 2019 04:17:12 +0200 Subject: Fix remote and staff-removed statuses leaving media behind for a day (#11638) The reason for unattaching media instead of removing it is to support delete & redraft functionality, but remote or staff-removed statuses will never be redrafted, so the media should be deleted immediately --- app/controllers/api/v1/statuses_controller.rb | 2 +- app/lib/activitypub/activity/delete.rb | 2 +- app/models/form/status_batch.rb | 2 +- app/services/batched_remove_status_service.rb | 2 +- app/services/remove_status_service.rb | 12 ++++++++++++ app/workers/removal_worker.rb | 4 ++-- spec/controllers/admin/reported_statuses_controller_spec.rb | 2 +- spec/controllers/admin/statuses_controller_spec.rb | 2 +- spec/models/form/status_batch_spec.rb | 4 ++-- 9 files changed, 22 insertions(+), 10 deletions(-) (limited to 'app/workers') diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 71a505c26..39ca56482 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -53,7 +53,7 @@ class Api::V1::StatusesController < Api::BaseController @status = Status.where(account_id: current_user.account).find(params[:id]) authorize @status, :destroy? - RemovalWorker.perform_async(@status.id) + RemovalWorker.perform_async(@status.id, redraft: true) render json: @status, serializer: REST::StatusSerializer, source_requested: true end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 1f2b40c15..345060462 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -70,7 +70,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity end def delete_now! - RemoveStatusService.new.call(@status) + RemoveStatusService.new.call(@status, redraft: false) end def payload diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index 933dfdaca..831d8b7c5 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -34,7 +34,7 @@ class Form::StatusBatch def delete_statuses Status.where(id: status_ids).reorder(nil).find_each do |status| - RemovalWorker.perform_async(status.id) + RemovalWorker.perform_async(status.id, redraft: false) Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) log_action :destroy, status end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 6df8d4769..3638134be 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -8,7 +8,7 @@ class BatchedRemoveStatusService < BaseService # Dispatch Salmon deletes, unique per domain, of the deleted statuses, but only local ones # Remove statuses from home feeds # Push delete events to streaming API for home feeds and public feeds - # @param [Status] statuses A preferably batched array of statuses + # @param [Enumerable] statuses A preferably batched array of statuses # @param [Hash] options # @option [Boolean] :skip_side_effects def call(statuses, **options) diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 91c934181..685c1d4bf 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -4,6 +4,11 @@ class RemoveStatusService < BaseService include Redisable include Payloadable + # Delete a status + # @param [Status] status + # @param [Hash] options + # @option [Boolean] :redraft + # @options [Boolean] :original_removed def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @status = status @@ -24,6 +29,7 @@ class RemoveStatusService < BaseService remove_from_public remove_from_media if status.media_attachments.any? remove_from_spam_check + remove_media @status.destroy! else @@ -143,6 +149,12 @@ class RemoveStatusService < BaseService redis.publish('timeline:public:local:media', @payload) if @status.local? end + def remove_media + return if @options[:redraft] + + @status.media_attachments.destroy_all + end + def remove_from_spam_check redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id) end diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb index 19a660dd3..14423a4fb 100644 --- a/app/workers/removal_worker.rb +++ b/app/workers/removal_worker.rb @@ -3,8 +3,8 @@ class RemovalWorker include Sidekiq::Worker - def perform(status_id) - RemoveStatusService.new.call(Status.find(status_id)) + def perform(status_id, options = {}) + RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys) rescue ActiveRecord::RecordNotFound true end diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb index c358506d6..bd146b795 100644 --- a/spec/controllers/admin/reported_statuses_controller_spec.rb +++ b/spec/controllers/admin/reported_statuses_controller_spec.rb @@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 1a08c10b7..6b06343ef 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -65,7 +65,7 @@ describe Admin::StatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) end end diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb index 00c790a11..f9c58c90f 100644 --- a/spec/models/form/status_batch_spec.rb +++ b/spec/models/form/status_batch_spec.rb @@ -41,12 +41,12 @@ describe Form::StatusBatch do it 'call RemovalWorker' do form.save - expect(RemovalWorker).to have_received(:perform_async).with(status.id) + expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false) end it 'do not call RemovalWorker' do form.save - expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id) + expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false) end end end -- cgit From 282ea170782e4ce1ed5251a1b94857a512412397 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Aug 2019 21:55:56 +0200 Subject: Add soft delete for statuses for instant deletes through API (#11623) * Add soft delete for statuses to allow them to appear instant * Allow reporting soft-deleted statuses and show them in the admin UI * Change index for getting an account's statuses --- Gemfile | 1 + Gemfile.lock | 3 +++ app/controllers/api/v1/reports_controller.rb | 2 +- app/controllers/api/v1/statuses/reblogs_controller.rb | 3 ++- app/controllers/api/v1/statuses_controller.rb | 1 + app/models/form/status_batch.rb | 1 + app/models/report.rb | 2 +- app/models/status.rb | 6 +++++- app/views/admin/reports/_status.html.haml | 5 ++++- app/workers/removal_worker.rb | 2 +- config/locales/en.yml | 1 + db/migrate/20190819134503_add_deleted_at_to_statuses.rb | 5 +++++ db/migrate/20190820003045_update_statuses_index.rb | 13 +++++++++++++ db/schema.rb | 5 +++-- 14 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 db/migrate/20190819134503_add_deleted_at_to_statuses.rb create mode 100644 db/migrate/20190820003045_update_statuses_index.rb (limited to 'app/workers') diff --git a/Gemfile b/Gemfile index 250a28a3a..86dab965a 100644 --- a/Gemfile +++ b/Gemfile @@ -43,6 +43,7 @@ gem 'omniauth-cas', '~> 1.1' gem 'omniauth-saml', '~> 1.10' gem 'omniauth', '~> 1.9' +gem 'discard', '~> 1.1' gem 'doorkeeper', '~> 5.1' gem 'fast_blank', '~> 1.0' gem 'fastimage' diff --git a/Gemfile.lock b/Gemfile.lock index 1da6d73a6..b896909a4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -204,6 +204,8 @@ GEM devise (>= 4.0.0) rpam2 (~> 4.0) diff-lcs (1.3) + discard (1.1.0) + activerecord (>= 4.2, < 7) docile (1.3.2) domain_name (0.5.20180417) unf (>= 0.0.5, < 1.0.0) @@ -692,6 +694,7 @@ DEPENDENCIES devise (~> 4.6) devise-two-factor (~> 3.1) devise_pam_authenticatable2 (~> 9.2) + discard (~> 1.1) doorkeeper (~> 5.1) dotenv-rails (~> 2.7) fabrication (~> 2.20) diff --git a/app/controllers/api/v1/reports_controller.rb b/app/controllers/api/v1/reports_controller.rb index e182a9c6c..1b0b4b05b 100644 --- a/app/controllers/api/v1/reports_controller.rb +++ b/app/controllers/api/v1/reports_controller.rb @@ -21,7 +21,7 @@ class Api::V1::ReportsController < Api::BaseController private def reported_status_ids - reported_account.statuses.find(status_ids).pluck(:id) + reported_account.statuses.with_discarded.find(status_ids).pluck(:id) end def status_ids diff --git a/app/controllers/api/v1/statuses/reblogs_controller.rb b/app/controllers/api/v1/statuses/reblogs_controller.rb index ed4f55100..42381a37f 100644 --- a/app/controllers/api/v1/statuses/reblogs_controller.rb +++ b/app/controllers/api/v1/statuses/reblogs_controller.rb @@ -18,6 +18,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController @reblogs_map = { @status.id => false } authorize status_for_destroy, :unreblog? + status_for_destroy.discard RemovalWorker.perform_async(status_for_destroy.id) render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, reblogs_map: @reblogs_map) @@ -30,7 +31,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController end def status_for_destroy - current_user.account.statuses.where(reblog_of_id: params[:status_id]).first! + @status_for_destroy ||= current_user.account.statuses.where(reblog_of_id: params[:status_id]).first! end def reblog_params diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 39ca56482..bba3c0651 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -53,6 +53,7 @@ class Api::V1::StatusesController < Api::BaseController @status = Status.where(account_id: current_user.account).find(params[:id]) authorize @status, :destroy? + @status.discard RemovalWorker.perform_async(@status.id, redraft: true) render json: @status, serializer: REST::StatusSerializer, source_requested: true diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index 831d8b7c5..e09cc2594 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -34,6 +34,7 @@ class Form::StatusBatch def delete_statuses Status.where(id: status_ids).reorder(nil).find_each do |status| + status.discard RemovalWorker.perform_async(status.id, redraft: false) Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) log_action :destroy, status diff --git a/app/models/report.rb b/app/models/report.rb index 5192ceef7..1e707ff1c 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -43,7 +43,7 @@ class Report < ApplicationRecord end def statuses - Status.where(id: status_ids).includes(:account, :media_attachments, :mentions) + Status.with_discarded.where(id: status_ids).includes(:account, :media_attachments, :mentions) end def media_attachments diff --git a/app/models/status.rb b/app/models/status.rb index 0538c4e9e..9cfaddcec 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -22,15 +22,19 @@ # application_id :bigint(8) # in_reply_to_account_id :bigint(8) # poll_id :bigint(8) +# deleted_at :datetime # class Status < ApplicationRecord before_destroy :unlink_from_conversations + include Discard::Model include Paginable include Cacheable include StatusThreadingConcern + self.discard_column = :deleted_at + # If `override_timestamps` is set at creation time, Snowflake ID creation # will be based on current time instead of `created_at` attr_accessor :override_timestamps @@ -72,7 +76,7 @@ class Status < ApplicationRecord accepts_nested_attributes_for :poll - default_scope { recent } + default_scope { recent.kept } scope :recent, -> { reorder(id: :desc) } scope :remote, -> { where(local: false).where.not(uri: nil) } diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml index 9376db7ff..6facc0a56 100644 --- a/app/views/admin/reports/_status.html.haml +++ b/app/views/admin/reports/_status.html.haml @@ -16,11 +16,14 @@ - video = status.proper.media_attachments.first = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description - else - = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } + = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } .detailed-status__meta = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) + - if status.discarded? + · + %span.negative-hint= t('admin.statuses.deleted') · - if status.reblog? = fa_icon('retweet fw') diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb index 14423a4fb..2a1eaa89b 100644 --- a/app/workers/removal_worker.rb +++ b/app/workers/removal_worker.rb @@ -4,7 +4,7 @@ class RemovalWorker include Sidekiq::Worker def perform(status_id, options = {}) - RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys) + RemoveStatusService.new.call(Status.with_discarded.find(status_id), **options.symbolize_keys) rescue ActiveRecord::RecordNotFound true end diff --git a/config/locales/en.yml b/config/locales/en.yml index 8d267065c..a50dcb8a5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -499,6 +499,7 @@ en: delete: Delete nsfw_off: Mark as not sensitive nsfw_on: Mark as sensitive + deleted: Deleted failed_to_execute: Failed to execute media: title: Media diff --git a/db/migrate/20190819134503_add_deleted_at_to_statuses.rb b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb new file mode 100644 index 000000000..5af109097 --- /dev/null +++ b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb @@ -0,0 +1,5 @@ +class AddDeletedAtToStatuses < ActiveRecord::Migration[5.2] + def change + add_column :statuses, :deleted_at, :datetime + end +end diff --git a/db/migrate/20190820003045_update_statuses_index.rb b/db/migrate/20190820003045_update_statuses_index.rb new file mode 100644 index 000000000..5c2ea1f6a --- /dev/null +++ b/db/migrate/20190820003045_update_statuses_index.rb @@ -0,0 +1,13 @@ +class UpdateStatusesIndex < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def up + safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], where: 'deleted_at IS NULL', order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20190820 } + remove_index :statuses, name: :index_statuses_20180106 + end + + def down + safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20180106 } + remove_index :statuses, name: :index_statuses_20190820 + end +end diff --git a/db/schema.rb b/db/schema.rb index 18f615d61..afa6d724c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_15_225426) do +ActiveRecord::Schema.define(version: 2019_08_20_003045) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -644,7 +644,8 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do t.bigint "application_id" t.bigint "in_reply_to_account_id" t.bigint "poll_id" - t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20180106", order: { id: :desc } + t.datetime "deleted_at" + t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)" t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id" t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id" t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id" -- cgit From 70ddef2654a931827ce5e4323e3042365f6078f2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 2 Sep 2019 18:11:13 +0200 Subject: Change trending hashtags to not disappear instantly after midnight (#11712) --- app/controllers/admin/tags_controller.rb | 2 +- app/lib/feed_manager.rb | 2 +- app/models/tag.rb | 4 +- app/models/trending_tags.rb | 102 +++++++++++++++------ app/workers/scheduler/trending_tags_scheduler.rb | 11 +++ config/sidekiq.yml | 3 + db/migrate/20190901035623_add_max_score_to_tags.rb | 6 ++ .../20190901040524_remove_score_from_tags.rb | 12 +++ db/schema.rb | 6 +- spec/models/trending_tags_spec.rb | 68 ++++++++++++++ 10 files changed, 179 insertions(+), 37 deletions(-) create mode 100644 app/workers/scheduler/trending_tags_scheduler.rb create mode 100644 db/migrate/20190901035623_add_max_score_to_tags.rb create mode 100644 db/post_migrate/20190901040524_remove_score_from_tags.rb create mode 100644 spec/models/trending_tags_spec.rb (limited to 'app/workers') diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index 25d9b7d3d..8bd4e5f8b 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -57,7 +57,7 @@ module Admin scope = scope.unreviewed if filter_params[:review] == 'unreviewed' scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed' scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review' - scope.order(score: :desc) + scope.order(max_score: :desc) end def filter_params diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index ca3d890a8..871ec5c19 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -63,7 +63,7 @@ class FeedManager reblog_key = key(type, account_id, 'reblogs') # Remove any items past the MAX_ITEMS'th entry in our feed - redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) + redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1)) # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # tracking anything after it for deduplication purposes. diff --git a/app/models/tag.rb b/app/models/tag.rb index 945e3a3c6..135e0a030 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -7,14 +7,14 @@ # name :string default(""), not null # created_at :datetime not null # updated_at :datetime not null -# score :integer # usable :boolean # trendable :boolean # listable :boolean # reviewed_at :datetime # requested_review_at :datetime # last_status_at :datetime -# last_trend_at :datetime +# max_score :float +# max_score_at :datetime # class Tag < ApplicationRecord diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index e4ce988c1..e1b92b175 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -7,6 +7,8 @@ class TrendingTags THRESHOLD = 5 LIMIT = 10 REVIEW_THRESHOLD = 3 + MAX_SCORE_COOLDOWN = 3.days.freeze + MAX_SCORE_HALFLIFE = 6.hours.freeze class << self include Redisable @@ -16,14 +18,75 @@ class TrendingTags increment_historical_use!(tag.id, at_time) increment_unique_use!(tag.id, account.id, at_time) - increment_vote!(tag, at_time) + increment_use!(tag.id, at_time) tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago - tag.update(last_trend_at: Time.now.utc) if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago) + end + + def update!(at_time = Time.now.utc) + tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) + tags = Tag.where(id: tag_ids.uniq) + + # First pass to calculate scores and update the set + + tags.each do |tag| + expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f + expected = 1.0 if expected.zero? + observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) + + score = begin + if expected > observed || observed < THRESHOLD + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) + + if decaying_score.zero? + redis.zrem(KEY, tag.id) + else + redis.zadd(KEY, decaying_score, tag.id) + end + end + + users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) + + # Second pass to notify about previously unreviewed trends + + tags.each do |tag| + current_rank = redis.zrevrank(KEY, tag.id) + needs_review_notification = tag.requires_review? && !tag.requested_review? + rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD + + next unless !tag.trendable? && rank_passes_threshold && needs_review_notification + + tag.touch(:requested_review_at) + + users_for_review.each do |user| + AdminMailer.new_trending_tag(user.account, tag).deliver_later! + end + end + + # Trim older items + + redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) end def get(limit, filtered: true) - tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i) + tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) tags = Tag.where(id: tag_ids) tags = tags.where(trendable: true) if filtered @@ -33,8 +96,8 @@ class TrendingTags end def trending?(tag) - rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id) - rank.present? && rank <= LIMIT + rank = redis.zrevrank(KEY, tag.id) + rank.present? && rank < LIMIT end private @@ -51,31 +114,10 @@ class TrendingTags redis.expire(key, EXPIRE_HISTORY_AFTER) end - def increment_vote!(tag, at_time) - key = "#{KEY}:#{at_time.beginning_of_day.to_i}" - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - - if expected > observed || observed < THRESHOLD - redis.zrem(key, tag.id) - else - score = ((observed - expected)**2) / expected - old_rank = redis.zrevrank(key, tag.id) - - redis.zadd(key, score, tag.id) - request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review? - end - - redis.expire(key, EXPIRE_TRENDS_AFTER) - end - - def request_review!(tag) - return unless Setting.trends - - tag.touch(:requested_review_at) - - User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? } + def increment_use!(tag_id, at_time) + key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" + redis.sadd(key, tag_id) + redis.expire(key, EXPIRE_HISTORY_AFTER) end end end diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb new file mode 100644 index 000000000..77f0d5747 --- /dev/null +++ b/app/workers/scheduler/trending_tags_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::TrendingTagsScheduler + include Sidekiq::Worker + + sidekiq_options unique: :until_executed, retry: 0 + + def perform + TrendingTags.update! if Setting.trends + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 6ebe450b0..5de25de23 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -9,6 +9,9 @@ scheduled_statuses_scheduler: every: '5m' class: Scheduler::ScheduledStatusesScheduler + trending_tags_scheduler: + every: '5m' + class: Scheduler::TrendingTagsScheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::MediaCleanupScheduler diff --git a/db/migrate/20190901035623_add_max_score_to_tags.rb b/db/migrate/20190901035623_add_max_score_to_tags.rb new file mode 100644 index 000000000..f936e9871 --- /dev/null +++ b/db/migrate/20190901035623_add_max_score_to_tags.rb @@ -0,0 +1,6 @@ +class AddMaxScoreToTags < ActiveRecord::Migration[5.2] + def change + add_column :tags, :max_score, :float + add_column :tags, :max_score_at, :datetime + end +end diff --git a/db/post_migrate/20190901040524_remove_score_from_tags.rb b/db/post_migrate/20190901040524_remove_score_from_tags.rb new file mode 100644 index 000000000..a1112700b --- /dev/null +++ b/db/post_migrate/20190901040524_remove_score_from_tags.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RemoveScoreFromTags < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + safety_assured do + remove_column :tags, :score, :int + remove_column :tags, :last_trend_at, :datetime + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 482bca367..5576f70bf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_23_221802) do +ActiveRecord::Schema.define(version: 2019_09_01_040524) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -664,14 +664,14 @@ ActiveRecord::Schema.define(version: 2019_08_23_221802) do t.string "name", default: "", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "score" t.boolean "usable" t.boolean "trendable" t.boolean "listable" t.datetime "reviewed_at" t.datetime "requested_review_at" t.datetime "last_status_at" - t.datetime "last_trend_at" + t.float "max_score" + t.datetime "max_score_at" t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true end diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb new file mode 100644 index 000000000..b6122c994 --- /dev/null +++ b/spec/models/trending_tags_spec.rb @@ -0,0 +1,68 @@ +require 'rails_helper' + +RSpec.describe TrendingTags do + describe '.record_use!' do + pending + end + + describe '.update!' do + let!(:at_time) { Time.now.utc } + let!(:tag1) { Fabricate(:tag, name: 'Catstodon') } + let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') } + let!(:tag3) { Fabricate(:tag, name: 'OCs') } + + before do + allow(Redis.current).to receive(:pfcount) do |key| + case key + when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 2 + when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts" + 16 + when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 0 + when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts" + 4 + when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 13 + end + end + + Redis.current.zadd('trending_tags', 0.9, tag3.id) + Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id]) + + tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours) + + described_class.update!(at_time) + end + + it 'calculates and re-calculates scores' do + expect(described_class.get(10, filtered: false)).to eq [tag1, tag3] + end + + it 'omits hashtags below threshold' do + expect(described_class.get(10, filtered: false)).to_not include(tag2) + end + + it 'decays scores' do + expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9 + end + end + + describe '.trending?' do + let(:tag) { Fabricate(:tag) } + + before do + 10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) } + end + + it 'returns true if the hashtag is within limit' do + Redis.current.zadd('trending_tags', 11, tag.id) + expect(described_class.trending?(tag)).to be true + end + + it 'returns false if the hashtag is outside the limit' do + Redis.current.zadd('trending_tags', 0, tag.id) + expect(described_class.trending?(tag)).to be false + end + end +end -- cgit From 86748148256b504c0411119628435b1445959309 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 10 Sep 2019 13:48:48 +0200 Subject: Change tootctl to use inline parallelization instead of Sidekiq (#11776) - Remove --background option - Add --concurrency(=5) option - Add progress bars --- Gemfile | 1 + Gemfile.lock | 1 + app/models/media_attachment.rb | 1 + app/models/preview_card.rb | 2 + app/workers/maintenance/destroy_media_worker.rb | 14 -- .../maintenance/redownload_account_media_worker.rb | 16 -- app/workers/maintenance/uncache_media_worker.rb | 18 -- app/workers/maintenance/uncache_preview_worker.rb | 18 -- lib/mastodon/accounts_cli.rb | 192 +++++++++------------ lib/mastodon/cache_cli.rb | 16 +- lib/mastodon/cli_helper.rb | 49 ++++++ lib/mastodon/domains_cli.rb | 27 ++- lib/mastodon/feeds_cli.rb | 42 +---- lib/mastodon/media_cli.rb | 57 ++---- lib/mastodon/preview_cards_cli.rb | 83 +++------ 15 files changed, 200 insertions(+), 337 deletions(-) delete mode 100644 app/workers/maintenance/destroy_media_worker.rb delete mode 100644 app/workers/maintenance/redownload_account_media_worker.rb delete mode 100644 app/workers/maintenance/uncache_media_worker.rb delete mode 100644 app/workers/maintenance/uncache_preview_worker.rb (limited to 'app/workers') diff --git a/Gemfile b/Gemfile index 73edb2a6a..af0e8e2fc 100644 --- a/Gemfile +++ b/Gemfile @@ -77,6 +77,7 @@ gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.1', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'rqrcode', '~> 0.10' +gem 'ruby-progressbar', '~> 1.10' gem 'sanitize', '~> 5.1' gem 'sidekiq', '~> 5.2' gem 'sidekiq-scheduler', '~> 3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 125cca4ba..6e931c611 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -769,6 +769,7 @@ DEPENDENCIES rspec-sidekiq (~> 3.0) rubocop (~> 0.74) rubocop-rails (~> 2.3) + ruby-progressbar (~> 1.10) sanitize (~> 5.1) sidekiq (~> 5.2) sidekiq-bulk (~> 0.2.0) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 44f8e6be6..b58025015 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -129,6 +129,7 @@ class MediaAttachment < ApplicationRecord scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) } scope :local, -> { where(remote_url: '') } scope :remote, -> { where.not(remote_url: '') } + scope :cached, -> { remote.where.not(file_file_name: nil) } default_scope { order(id: :asc) } diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index a792b352b..9d6c1938a 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -43,6 +43,8 @@ class PreviewCard < ApplicationRecord validates_attachment_size :image, less_than: LIMIT remotable_attachment :image, LIMIT + scope :cached, -> { where.not(image_file_name: [nil, '']) } + before_save :extract_dimensions, if: :link? def save_with_optional_image! diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb deleted file mode 100644 index cde33d6d7..000000000 --- a/app/workers/maintenance/destroy_media_worker.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::DestroyMediaWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull' - - def perform(media_attachment_id) - media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) - media.destroy - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb deleted file mode 100644 index 6afbe6e19..000000000 --- a/app/workers/maintenance/redownload_account_media_worker.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::RedownloadAccountMediaWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull', retry: false - - def perform(account_id) - account = account_id.is_a?(Account) ? account_id : Account.find(account_id) - account.reset_avatar! - account.reset_header! - account.save - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb deleted file mode 100644 index 4bc62ef75..000000000 --- a/app/workers/maintenance/uncache_media_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::UncacheMediaWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull' - - def perform(media_attachment_id) - media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) - - return if media.file.blank? - - media.file.destroy - media.save - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/app/workers/maintenance/uncache_preview_worker.rb b/app/workers/maintenance/uncache_preview_worker.rb deleted file mode 100644 index 810ffd8cc..000000000 --- a/app/workers/maintenance/uncache_preview_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::UncachePreviewWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull' - - def perform(preview_card_id) - preview_card = PreviewCard.find(preview_card_id) - - return if preview_card.image.blank? - - preview_card.image.destroy - preview_card.save - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index d1854acc0..b16bf2e38 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -7,6 +7,8 @@ require_relative 'cli_helper' module Mastodon class AccountsCLI < Thor + include CLIHelper + def self.exit_on_failure? true end @@ -26,18 +28,20 @@ module Mastodon if options[:all] processed = 0 delay = 0 + scope = Account.local.without_suspended + progress = create_progress_bar(scope.count) - Account.local.without_suspended.find_in_batches do |accounts| + scope.find_in_batches do |accounts| accounts.each do |account| rotate_keys_for_account(account, delay) + progress.increment processed += 1 - say('.', :green, false) end delay += 5.minutes end - say + progress.finish say("OK, rotated keys for #{processed} accounts", :green) elsif username.present? rotate_keys_for_account(Account.find_local(username)) @@ -206,6 +210,8 @@ module Mastodon say('OK', :green) end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean desc 'cull', 'Remove remote accounts that no longer exist' long_desc <<-LONG_DESC @@ -215,63 +221,45 @@ module Mastodon Accounts that have had confirmed activity within the last week are excluded from the checks. - - Domains that are unreachable are not checked. - - With the --dry-run option, no deletes will actually be carried - out. LONG_DESC def cull skip_threshold = 7.days.ago - culled = 0 - dry_run_culled = [] - skip_domains = Set.new dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + skip_domains = Concurrent::Set.new - Account.remote.where(protocol: :activitypub).partitioned.find_each do |account| - next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) + processed, culled = parallelize_with_progress(Account.remote.where(protocol: :activitypub).partitioned) do |account| + next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) || skip_domains.include?(account.domain) code = 0 - unless skip_domains.include?(account.domain) - begin - code = Request.new(:head, account.uri).perform(&:code) - rescue HTTP::ConnectionError - skip_domains << account.domain - rescue StandardError - next - end + + begin + code = Request.new(:head, account.uri).perform(&:code) + rescue HTTP::ConnectionError + skip_domains << account.domain end if [404, 410].include?(code) - if options[:dry_run] - dry_run_culled << account.acct - else - SuspendAccountService.new.call(account, destroy: true) - end - culled += 1 - say('+', :green, false) + SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + 1 else - account.touch # Touch account even during dry run to avoid getting the account into the window again - say('.', nil, false) + # Touch account even during dry run to avoid getting the account into the window again + account.touch end end - say - say("Removed #{culled} accounts. #{skip_domains.size} servers skipped#{dry_run}", skip_domains.empty? ? :green : :yellow) + say("Visited #{processed} accounts, removed #{culled}#{dry_run}", :green) unless skip_domains.empty? - say('The following servers were not available during the check:', :yellow) + say('The following domains were not available during the check:', :yellow) skip_domains.each { |domain| say(' ' + domain) } end - - unless dry_run_culled.empty? - say('The following accounts would have been deleted:', :green) - dry_run_culled.each { |account| say(' ' + account) } - end end option :all, type: :boolean option :domain + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] + option :dry_run, type: :boolean desc 'refresh [USERNAME]', 'Fetch remote user data and files' long_desc <<-LONG_DESC Fetch remote user data and files for one or multiple accounts. @@ -280,21 +268,23 @@ module Mastodon Through the --domain option, this can be narrowed down to a specific domain only. Otherwise, a single remote account must be specified with USERNAME. - - All processing is done in the background through Sidekiq. LONG_DESC def refresh(username = nil) + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + if options[:domain] || options[:all] - queued = 0 scope = Account.remote scope = scope.where(domain: options[:domain]) if options[:domain] - scope.select(:id).reorder(nil).find_in_batches do |accounts| - Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts.map(&:id)) - queued += accounts.size + processed, = parallelize_with_progress(scope) do |account| + next if options[:dry_run] + + account.reset_avatar! + account.reset_header! + account.save end - say("Scheduled refreshment of #{queued} accounts", :green, true) + say("Refreshed #{processed} accounts#{dry_run}", :green, true) elsif username.present? username, domain = username.split('@') account = Account.find_remote(username, domain) @@ -304,76 +294,53 @@ module Mastodon exit(1) end - Maintenance::RedownloadAccountMediaWorker.perform_async(account.id) - say('OK', :green) + unless options[:dry_run] + account.reset_avatar! + account.reset_header! + account.save + end + + say("OK#{dry_run}", :green) else say('No account(s) given', :red) exit(1) end end - desc 'follow ACCT', 'Make all local accounts follow account specified by ACCT' - long_desc <<-LONG_DESC - Make all local accounts follow another local account specified by ACCT. - ACCT should be the username only. - LONG_DESC - def follow(acct) - if acct.include? '@' - say('Target account name should not contain a target instance, since it has to be a local account.', :red) - exit(1) - end - - target_account = ResolveAccountService.new.call(acct) - processed = 0 - failed = 0 + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] + desc 'follow USERNAME', 'Make all local accounts follow account specified by USERNAME' + def follow(username) + target_account = Account.find_local(username) if target_account.nil? - say("Target account (#{acct}) could not be resolved", :red) + say('No such account', :red) exit(1) end - Account.local.without_suspended.find_each do |account| - begin - FollowService.new.call(account, target_account) - processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) - end + processed, = parallelize_with_progress(Account.local.without_suspended) do |account| + FollowService.new.call(account, target_account) end - say("OK, followed target from #{processed} accounts, skipped #{failed}", :green) + say("OK, followed target from #{processed} accounts", :green) end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT' - long_desc <<-LONG_DESC - Make all local accounts unfollow an account specified by ACCT. ACCT can be - a simple username, in case of a local user. It can also be in the format - username@domain, in case of a remote user. - LONG_DESC def unfollow(acct) target_account = Account.find_remote(*acct.split('@')) - processed = 0 - failed = 0 if target_account.nil? - say("Target account (#{acct}) was not found", :red) + say('No such account', :red) exit(1) end - target_account.followers.local.find_each do |account| - begin - UnfollowService.new.call(account, target_account) - processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) - end + parallelize_with_progress(target_account.followers.local) do |account| + UnfollowService.new.call(account, target_account) end - say("OK, unfollowed target from #{processed} accounts, skipped #{failed}", :green) + say("OK, unfollowed target from #{processed} accounts", :green) end option :follows, type: :boolean, default: false @@ -396,51 +363,50 @@ module Mastodon account = Account.find_local(username) if account.nil? - say('No user with such username', :red) + say('No such account', :red) exit(1) end - if options[:follows] - processed = 0 - failed = 0 + total = 0 + total += Account.where(id: ::Follow.where(account: account).select(:target_account_id)).count if options[:follows] + total += Account.where(id: ::Follow.where(target_account: account).select(:account_id)).count if options[:followers] + progress = create_progress_bar(total) + processed = 0 - say("Unfollowing #{account.username}'s followees, this might take a while...") + if options[:follows] + scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id)) - Account.where(id: ::Follow.where(account: account).select(:target_account_id)).find_each do |target_account| + scope.find_each do |target_account| begin UnfollowService.new.call(account, target_account) + rescue => e + progress.log pastel.red("Error processing #{target_account.id}: #{e}") + ensure + progress.increment processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) end end BootstrapTimelineWorker.perform_async(account.id) - - say("OK, unfollowed #{processed} followees, skipped #{failed}", :green) end if options[:followers] - processed = 0 - failed = 0 - - say("Removing #{account.username}'s followers, this might take a while...") + scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id)) - Account.where(id: ::Follow.where(target_account: account).select(:account_id)).find_each do |target_account| + scope.find_each do |target_account| begin UnfollowService.new.call(target_account, account) + rescue => e + progress.log pastel.red("Error processing #{target_account.id}: #{e}") + ensure + progress.increment processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) end end - - say("OK, removed #{processed} followers, skipped #{failed}", :green) end + + progress.finish + say("Processed #{processed} relationships", :green, true) end option :number, type: :numeric, aliases: [:n] diff --git a/lib/mastodon/cache_cli.rb b/lib/mastodon/cache_cli.rb index 5b0eea91b..803404c34 100644 --- a/lib/mastodon/cache_cli.rb +++ b/lib/mastodon/cache_cli.rb @@ -6,6 +6,8 @@ require_relative 'cli_helper' module Mastodon class CacheCLI < Thor + include CLIHelper + def self.exit_on_failure? true end @@ -16,6 +18,8 @@ module Mastodon say('OK', :green) end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] desc 'recount TYPE', 'Update hard-cached counters' long_desc <<~LONG_DESC Update hard-cached counters of TYPE by counting referenced @@ -25,32 +29,24 @@ module Mastodon size of the database. LONG_DESC def recount(type) - processed = 0 - case type when 'accounts' - Account.local.includes(:account_stat).find_each do |account| + processed, = parallelize_with_progress(Account.local.includes(:account_stat)) do |account| account_stat = account.account_stat account_stat.following_count = account.active_relationships.count account_stat.followers_count = account.passive_relationships.count account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count account_stat.save if account_stat.changed? - - processed += 1 - say('.', :green, false) end when 'statuses' - Status.includes(:status_stat).find_each do |status| + processed, = parallelize_with_progress(Status.includes(:status_stat)) do |status| status_stat = status.status_stat status_stat.replies_count = status.replies.where.not(visibility: :direct).count status_stat.reblogs_count = status.reblogs.count status_stat.favourites_count = status.favourites.count status_stat.save if status_stat.changed? - - processed += 1 - say('.', :green, false) end else say("Unknown type: #{type}", :red) diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index 2f807d08c..da7348349 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -7,3 +7,52 @@ ActiveRecord::Base.logger = dev_null ActiveJob::Base.logger = dev_null HttpLog.configuration.logger = dev_null Paperclip.options[:log] = false + +module Mastodon + module CLIHelper + def create_progress_bar(total = nil) + ProgressBar.create(total: total, format: '%c/%u |%b%i| %e') + end + + def parallelize_with_progress(scope) + ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + + progress = create_progress_bar(scope.count) + pool = Concurrent::FixedThreadPool.new(options[:concurrency]) + total = Concurrent::AtomicFixnum.new(0) + aggregate = Concurrent::AtomicFixnum.new(0) + + scope.reorder(nil).find_in_batches do |items| + futures = [] + + items.each do |item| + futures << Concurrent::Future.execute(executor: pool) do + ActiveRecord::Base.connection_pool.with_connection do + begin + progress.log("Processing #{item.id}") if options[:verbose] + + result = yield(item) + aggregate.increment(result) if result.is_a?(Integer) + rescue => e + progress.log pastel.red("Error processing #{item.id}: #{e}") + ensure + progress.increment + end + end + end + end + + total.increment(items.size) + futures.map(&:value) + end + + progress.finish + + [total.value, aggregate.value] + end + + def pastel + @pastel ||= Pastel.new + end + end +end diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 17cafd1bc..c612c2d72 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -7,10 +7,14 @@ require_relative 'cli_helper' module Mastodon class DomainsCLI < Thor + include CLIHelper + def self.exit_on_failure? true end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean option :whitelist_mode, type: :boolean desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace' @@ -24,7 +28,6 @@ module Mastodon are removed from the database. LONG_DESC def purge(domain = nil) - removed = 0 dry_run = options[:dry_run] ? ' (DRY RUN)' : '' scope = begin @@ -38,25 +41,22 @@ module Mastodon end end - scope.find_each do |account| + processed, = parallelize_with_progress(scope) do |account| SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] - removed += 1 - say('.', :green, false) end DomainBlock.where(domain: domain).destroy_all unless options[:dry_run] - say - say("Removed #{removed} accounts#{dry_run}", :green) + say("Removed #{processed} accounts#{dry_run}", :green) custom_emojis = CustomEmoji.where(domain: domain) custom_emojis_count = custom_emojis.count custom_emojis.destroy_all unless options[:dry_run] + say("Removed #{custom_emojis_count} custom emojis", :green) end option :concurrency, type: :numeric, default: 50, aliases: [:c] - option :silent, type: :boolean, default: false, aliases: [:s] option :format, type: :string, default: 'summary', aliases: [:f] option :exclude_suspended, type: :boolean, default: false, aliases: [:x] desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START' @@ -69,8 +69,6 @@ module Mastodon The --concurrency (-c) option controls the number of threads performing HTTP requests at the same time. More threads means the crawl may complete faster. - The --silent (-s) option controls progress output. - The --format (-f) option controls how the data is displayed at the end. By default (`summary`), a summary of the statistics is returned. The other options are `domains`, which returns a newline-delimited list of all discovered peers, @@ -87,6 +85,7 @@ module Mastodon start_at = Time.now.to_f seed = start ? [start] : Account.remote.domains blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$') + progress = create_progress_bar pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0) @@ -95,7 +94,6 @@ module Mastodon next if options[:exclude_suspended] && domain.match(blocked_domains) stats[domain] = nil - processed.increment begin Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res| @@ -115,11 +113,11 @@ module Mastodon next unless res.code == 200 stats[domain]['activity'] = Oj.load(res.to_s) end - - say('.', :green, false) unless options[:silent] rescue StandardError failed.increment - say('.', :red, false) unless options[:silent] + ensure + processed.increment + progress.increment unless progress.finished? end end @@ -133,10 +131,9 @@ module Mastodon pool.shutdown pool.wait_for_termination(20) ensure + progress.finish pool.shutdown - say unless options[:silent] - case options[:format] when 'summary' stats_to_summary(stats, processed, failed, start_at) diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb index fe11c3df4..ea7c90dff 100644 --- a/lib/mastodon/feeds_cli.rb +++ b/lib/mastodon/feeds_cli.rb @@ -6,55 +6,33 @@ require_relative 'cli_helper' module Mastodon class FeedsCLI < Thor + include CLIHelper + def self.exit_on_failure? true end option :all, type: :boolean, default: false - option :background, type: :boolean, default: false + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean, default: false - option :verbose, type: :boolean, default: false desc 'build [USERNAME]', 'Build home and list feeds for one or all users' long_desc <<-LONG_DESC Build home and list feeds that are stored in Redis from the database. With the --all option, all active users will be processed. Otherwise, a single user specified by USERNAME. - - With the --background option, regeneration will be queued into Sidekiq, - and the command will exit as soon as possible. - - With the --dry-run option, no work will be done. - - With the --verbose option, when accounts are processed sequentially in the - foreground, the IDs of the accounts will be printed. LONG_DESC def build(username = nil) dry_run = options[:dry_run] ? '(DRY RUN)' : '' if options[:all] || username.nil? - processed = 0 - queued = 0 - User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users| - if options[:background] - RegenerationWorker.push_bulk(users.map(&:account_id)) unless options[:dry_run] - queued += users.size - else - users.each do |user| - RegenerationWorker.new.perform(user.account_id) unless options[:dry_run] - options[:verbose] ? say(user.account_id) : say('.', :green, false) - processed += 1 - end - end + processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account| + PrecomputeFeedService.new.call(account) unless options[:dry_run] end - if options[:background] - say("Scheduled feed regeneration for #{queued} accounts #{dry_run}", :green, true) - else - say - say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true) - end + say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true) elsif username.present? account = Account.find_local(username) @@ -63,11 +41,7 @@ module Mastodon exit(1) end - if options[:background] - RegenerationWorker.perform_async(account.id) unless options[:dry_run] - else - RegenerationWorker.new.perform(account.id) unless options[:dry_run] - end + PrecomputeFeedService.new.call(account) unless options[:dry_run] say("OK #{dry_run}", :green, true) else diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 6152d5a09..0659b6b65 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -7,14 +7,15 @@ require_relative 'cli_helper' module Mastodon class MediaCLI < Thor include ActionView::Helpers::NumberHelper + include CLIHelper def self.exit_on_failure? true end - option :days, type: :numeric, default: 7 - option :background, type: :boolean, default: false - option :verbose, type: :boolean, default: false + option :days, type: :numeric, default: 7, aliases: [:d] + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false desc 'remove', 'Remove remote media files' long_desc <<-DESC @@ -22,49 +23,25 @@ module Mastodon The --days option specifies how old media attachments have to be before they are removed. It defaults to 7 days. - - With the --background option, instead of deleting the files sequentially, - they will be queued into Sidekiq and the command will exit as soon as - possible. In Sidekiq they will be processed with higher concurrency, but - it may impact other operations of the Mastodon server, and it may overload - the underlying file storage. - - With the --dry-run option, no work will be done. - - With the --verbose option, when media attachments are processed sequentially in the - foreground, the IDs of the media attachments will be printed. DESC def remove - time_ago = options[:days].days.ago - queued = 0 - processed = 0 - size = 0 - dry_run = options[:dry_run] ? '(DRY RUN)' : '' + time_ago = options[:days].days.ago + dry_run = options[:dry_run] ? '(DRY RUN)' : '' - if options[:background] - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id, :file_file_size).reorder(nil).find_in_batches do |media_attachments| - queued += media_attachments.size - size += media_attachments.reduce(0) { |sum, m| sum + (m.file_file_size || 0) } - Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) unless options[:dry_run] - end - else - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).reorder(nil).find_in_batches do |media_attachments| - media_attachments.each do |m| - size += m.file_file_size || 0 - Maintenance::UncacheMediaWorker.new.perform(m) unless options[:dry_run] - options[:verbose] ? say(m.id) : say('.', :green, false) - processed += 1 - end - end - end + processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| + next if media_attachment.file.blank? + + size = media_attachment.file_file_size - say + unless options[:dry_run] + media_attachment.file.destroy + media_attachment.save + end - if options[:background] - say("Scheduled the deletion of #{queued} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) - else - say("Removed #{processed} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + size end + + say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) end end end diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb index 465fe7d0b..cf4407250 100644 --- a/lib/mastodon/preview_cards_cli.rb +++ b/lib/mastodon/preview_cards_cli.rb @@ -8,87 +8,52 @@ require_relative 'cli_helper' module Mastodon class PreviewCardsCLI < Thor include ActionView::Helpers::NumberHelper + include CLIHelper def self.exit_on_failure? true end option :days, type: :numeric, default: 180 - option :background, type: :boolean, default: false - option :verbose, type: :boolean, default: false + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean, default: false option :link, type: :boolean, default: false desc 'remove', 'Remove preview cards' long_desc <<-DESC - Removes locally thumbnails for previews. + Removes local thumbnails for preview cards. The --days option specifies how old preview cards have to be before - they are removed. It defaults to 180 days. + they are removed. It defaults to 180 days. Since preview cards will + not be re-fetched unless the link is re-posted after 2 weeks from + last time, it is not recommended to delete preview cards within the + last 14 days. - With the --background option, instead of deleting the files sequentially, - they will be queued into Sidekiq and the command will exit as soon as - possible. In Sidekiq they will be processed with higher concurrency, but - it may impact other operations of the Mastodon server, and it may overload - the underlying file storage. - - With the --dry-run option, no work will be done. - - With the --verbose option, when preview cards are processed sequentially in the - foreground, the IDs of the preview cards will be printed. - - With the --link option, delete only link-type preview cards. + With the --link option, only link-type preview cards will be deleted, + leaving video and photo cards untouched. DESC def remove - prompt = TTY::Prompt.new - time_ago = options[:days].days.ago - queued = 0 - processed = 0 - size = 0 - dry_run = options[:dry_run] ? '(DRY RUN)' : '' - link = options[:link] ? 'link-type ' : '' - scope = PreviewCard.where.not(image_file_name: nil) - scope = scope.where.not(image_file_name: '') - scope = scope.where(type: :link) if options[:link] - scope = scope.where('updated_at < ?', time_ago) + time_ago = options[:days].days.ago + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + link = options[:link] ? 'link-type ' : '' + scope = PreviewCard.cached + scope = scope.where(type: :link) if options[:link] + scope = scope.where('updated_at < ?', time_ago) - if time_ago > 2.weeks.ago - prompt.say "\n" - prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.') - prompt.say "\n" + processed, aggregate = parallelize_with_progress(scope) do |preview_card| + next if preview_card.image.blank? - unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false) - prompt.say "\n" - prompt.warn 'Nothing execute. Bye!' - prompt.say "\n" - exit(1) - end - end + size = preview_card.image_file_size - if options[:background] - scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| - queued += preview_cards.size - size += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) } - Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run] + unless options[:dry_run] + preview_card.image.destroy + preview_card.save end - else - scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| - preview_cards.each do |p| - size += p.image_file_size || 0 - Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run] - options[:verbose] ? say(p.id) : say('.', :green, false) - processed += 1 - end - end + size end - say - - if options[:background] - say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) - else - say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) - end + say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) end end end -- cgit From 031ca25014e0ba88d3dcc3086947b41449a672e2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 10 Sep 2019 15:29:12 +0200 Subject: Add retry for failed media downloads and `tootctl media refresh` (#11775) --- app/lib/activitypub/activity/create.rb | 21 +++++++------ app/models/concerns/remotable.rb | 12 ++++---- app/models/media_attachment.rb | 2 +- app/workers/redownload_media_worker.rb | 19 ++++++++++++ lib/mastodon/media_cli.rb | 54 ++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 app/workers/redownload_media_worker.rb (limited to 'app/workers') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 000b77df5..dea7fd43c 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -189,22 +189,25 @@ class ActivityPub::Activity::Create < ActivityPub::Activity media_attachments = [] as_array(@object['attachment']).each do |attachment| - next if attachment['url'].blank? + next if attachment['url'].blank? || media_attachments.size >= 4 - href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) - media_attachments << media_attachment + begin + href = Addressable::URI.parse(attachment['url']).normalize.to_s + media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachments << media_attachment - next if unsupported_media_type?(attachment['mediaType']) || skip_download? + next if unsupported_media_type?(attachment['mediaType']) || skip_download? - media_attachment.file_remote_url = href - media_attachment.save + media_attachment.file_remote_url = href + media_attachment.save + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError + RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) + end end media_attachments rescue Addressable::URI::InvalidURIError => e - Rails.logger.debug e - + Rails.logger.debug "Invalid URL in attachment: #{e}" media_attachments end diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index 9372a963b..082302619 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -4,7 +4,7 @@ module Remotable extend ActiveSupport::Concern class_methods do - def remotable_attachment(attachment_name, limit) + def remotable_attachment(attachment_name, limit, suppress_errors: true) attribute_name = "#{attachment_name}_remote_url".to_sym method_name = "#{attribute_name}=".to_sym alt_method_name = "reset_#{attachment_name}!".to_sym @@ -22,7 +22,7 @@ module Remotable begin Request.new(:get, url).perform do |response| - next if response.code != 200 + raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code) content_type = parse_content_type(response.headers.get('content-type').last) extname = detect_extname_from_content_type(content_type) @@ -41,11 +41,11 @@ module Remotable self[attribute_name] = url if has_attribute?(attribute_name) end - rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e + Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" + raise e unless suppress_errors + rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" - nil - rescue Paperclip::Error, Mastodon::DimensionsValidationError => e - Rails.logger.debug "Error processing remote #{attachment_name}: #{e}" nil end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index b58025015..a2b73f150 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -118,7 +118,7 @@ class MediaAttachment < ApplicationRecord validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format? - remotable_attachment :file, VIDEO_LIMIT + remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false include Attachmentable diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb new file mode 100644 index 000000000..98e995918 --- /dev/null +++ b/app/workers/redownload_media_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RedownloadMediaWorker + include Sidekiq::Worker + include ExponentialBackoff + + sidekiq_options queue: 'pull', retry: 3 + + def perform(id) + media_attachment = MediaAttachment.find(id) + + return if media_attachment.remote_url.blank? + + media_attachment.reset_file! + media_attachment.save + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 0659b6b65..ec2f36c30 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -43,5 +43,59 @@ module Mastodon say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) end + + option :account, type: :string + option :domain, type: :string + option :status, type: :numeric + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, default: false, aliases: [:v] + option :dry_run, type: :boolean, default: false + desc 'refresh', 'Fetch remote media files' + long_desc <<-DESC + Re-downloads media attachments from other servers. You must specify the + source of media attachments with one of the following options: + + Use the --status option to download attachments from a specific status, + using the status local numeric ID. + + Use the --account option to download attachments from a specific account, + using username@domain handle of the account. + + Use the --domain option to download attachments from a specific domain. + DESC + def refresh + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + + if options[:status] + scope = MediaAttachment.where(status_id: options[:status]) + elsif options[:account] + username, domain = username.split('@') + account = Account.find_remote(username, domain) + + if account.nil? + say('No such account', :red) + exit(1) + end + + scope = MediaAttachment.where(account_id: account.id) + elsif options[:domain] + scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain])) + else + exit(1) + end + + processed, aggregate = parallelize_with_progress(scope) do |media_attachment| + next if media_attachment.remote_url.blank? + + unless options[:dry_run] + media_attachment.reset_file! + media_attachment.save + end + + media_attachment.file_file_size + end + + say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) + end end end -- cgit From c5d37f18cb3f4d6212fb8f3e1c4e1e027f677ec5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 11 Sep 2019 16:32:44 +0200 Subject: Change deletes to preserve soft-deleted statuses in unresolved reports (#11805) Change all account actions except "none" to resolve all unresolved reports Refactor `SuspendAccountService` to be more readable --- app/controllers/admin/accounts_controller.rb | 2 +- app/controllers/admin/report_notes_controller.rb | 9 ++-- .../api/v1/admin/accounts_controller.rb | 2 +- app/lib/activitypub/activity/delete.rb | 3 +- app/models/account.rb | 1 + app/models/admin/account_action.rb | 24 +++++++-- app/models/form/account_batch.rb | 2 +- app/models/form/status_batch.rb | 2 +- app/models/report.rb | 1 + app/models/status.rb | 4 ++ app/models/user.rb | 4 ++ app/services/block_domain_service.rb | 2 +- app/services/remove_status_service.rb | 7 +-- app/services/suspend_account_service.rb | 62 ++++++++++++++++------ app/services/unallow_domain_service.rb | 2 +- app/workers/admin/suspension_worker.rb | 2 +- lib/mastodon/accounts_cli.rb | 4 +- lib/mastodon/domains_cli.rb | 2 +- .../admin/reported_statuses_controller_spec.rb | 2 +- spec/controllers/admin/statuses_controller_spec.rb | 2 +- spec/models/form/status_batch_spec.rb | 4 +- 21 files changed, 98 insertions(+), 45 deletions(-) (limited to 'app/workers') diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb index 2fa1dfe5f..68b6352f8 100644 --- a/app/controllers/admin/accounts_controller.rb +++ b/app/controllers/admin/accounts_controller.rb @@ -41,7 +41,7 @@ module Admin def reject authorize @account.user, :reject? - SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true) + SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false) redirect_to admin_pending_accounts_path end diff --git a/app/controllers/admin/report_notes_controller.rb b/app/controllers/admin/report_notes_controller.rb index bcb3f2026..b816c5b5d 100644 --- a/app/controllers/admin/report_notes_controller.rb +++ b/app/controllers/admin/report_notes_controller.rb @@ -5,10 +5,10 @@ module Admin before_action :set_report_note, only: [:destroy] def create - authorize ReportNote, :create? + authorize :report_note, :create? @report_note = current_account.report_notes.new(resource_params) - @report = @report_note.report + @report = @report_note.report if @report_note.save if params[:create_and_resolve] @@ -26,9 +26,8 @@ module Admin redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg') else - @report_notes = @report.notes.latest - @report_history = @report.history - @form = Form::StatusBatch.new + @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at) + @form = Form::StatusBatch.new render template: 'admin/reports/show' end diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb index c306180ca..c35ea5ab2 100644 --- a/app/controllers/api/v1/admin/accounts_controller.rb +++ b/app/controllers/api/v1/admin/accounts_controller.rb @@ -58,7 +58,7 @@ class Api::V1::Admin::AccountsController < Api::BaseController def reject authorize @account.user, :reject? - SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true) + SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false) render json: @account, serializer: REST::Admin::AccountSerializer end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 345060462..dc9ff580c 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -13,8 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def delete_person lock_or_return("delete_in_progress:#{@account.id}") do - SuspendAccountService.new.call(@account) - @account.destroy! + SuspendAccountService.new.call(@account, reserve_username: false) end end diff --git a/app/models/account.rb b/app/models/account.rb index 8c9388b95..55fe53fae 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -115,6 +115,7 @@ class Account < ApplicationRecord :approved?, :pending?, :disabled?, + :unconfirmed_or_pending?, :role, :admin?, :moderator?, diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index c7da8b52c..b30a82369 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -83,19 +83,23 @@ class Admin::AccountAction # A log entry is only interesting if the warning contains # custom text from someone. Otherwise it's just noise. + log_action(:create, warning) if warning.text.present? end def process_reports! - return if report_id.blank? + # If we're doing "mark as resolved" on a single report, + # then we want to keep other reports open in case they + # contain new actionable information. + # + # Otherwise, we will mark all unresolved reports about + # the account as resolved. - authorize(report, :update?) + reports.each { |report| authorize(report, :update?) } - if type == 'none' + reports.each do |report| log_action(:resolve, report) report.resolve!(current_account) - else - Report.where(target_account: target_account).unresolved.update_all(action_taken: true, action_taken_by_account_id: current_account.id) end end @@ -141,6 +145,16 @@ class Admin::AccountAction @report.status_ids if @report && include_statuses end + def reports + @reports ||= begin + if type == 'none' && with_report? + [report] + else + Report.where(target_account: target_account).unresolved + end + end + end + def warning_preset @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present? end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index f1b7a4566..0b285fde9 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -69,6 +69,6 @@ class Form::AccountBatch records = accounts.includes(:user) records.each { |account| authorize(account.user, :reject?) } - .each { |account| SuspendAccountService.new.call(account, including_user: true, destroy: true, skip_distribution: true) } + .each { |account| SuspendAccountService.new.call(account, reserve_email: false, reserve_username: false) } end end diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index e09cc2594..c4943a7ea 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -35,7 +35,7 @@ class Form::StatusBatch def delete_statuses Status.where(id: status_ids).reorder(nil).find_each do |status| status.discard - RemovalWorker.perform_async(status.id, redraft: false) + RemovalWorker.perform_async(status.id, immediate: true) Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) log_action :destroy, status end diff --git a/app/models/report.rb b/app/models/report.rb index 1e707ff1c..fb2e040ee 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -59,6 +59,7 @@ class Report < ApplicationRecord end def resolve!(acting_account) + RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] } update!(action_taken: true, action_taken_by_account_id: acting_account.id) end diff --git a/app/models/status.rb b/app/models/status.rb index 9cfaddcec..471bb03b4 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -214,6 +214,10 @@ class Status < ApplicationRecord !sensitive? && with_media? end + def reported? + @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists? + end + def emojis return @emojis if defined?(@emojis) diff --git a/app/models/user.rb b/app/models/user.rb index 95f1d8fc5..78b82a68f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -171,6 +171,10 @@ class User < ApplicationRecord confirmed? && approved? && !disabled? && !account.suspended? end + def unconfirmed_or_pending? + !(confirmed? && approved?) + end + def inactive_message !approved? ? :pending : super end diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb index 0ec6be503..ae461abf2 100644 --- a/app/services/block_domain_service.rb +++ b/app/services/block_domain_service.rb @@ -53,7 +53,7 @@ class BlockDomainService < BaseService def suspend_accounts! blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account| - SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at) + SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at) end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 685c1d4bf..f9352ed3d 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -8,7 +8,8 @@ class RemoveStatusService < BaseService # @param [Status] status # @param [Hash] options # @option [Boolean] :redraft - # @options [Boolean] :original_removed + # @option [Boolean] :immediate + # @option [Boolean] :original_removed def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @status = status @@ -31,7 +32,7 @@ class RemoveStatusService < BaseService remove_from_spam_check remove_media - @status.destroy! + @status.destroy! if @options[:immediate] || !@status.reported? else raise Mastodon::RaceConditionError end @@ -150,7 +151,7 @@ class RemoveStatusService < BaseService end def remove_media - return if @options[:redraft] + return if @options[:redraft] || (!@options[:immediate] && @status.reported?) @status.media_attachments.destroy_all end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 85da7e921..ecc893931 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -15,7 +15,6 @@ class SuspendAccountService < BaseService favourites follow_requests list_accounts - media_attachments mute_relationships muted_by_relationships notifications @@ -32,14 +31,26 @@ class SuspendAccountService < BaseService targeted_reports ).freeze - # Suspend an account and remove as much of its data as possible + # Suspend or remove an account and remove as much of its data + # as possible. If it's a local account and it has not been confirmed + # or never been approved, then side effects are skipped and both + # the user and account records are removed fully. Otherwise, + # it is controlled by options. # @param [Account] # @param [Hash] options - # @option [Boolean] :including_user Remove the user record as well - # @option [Boolean] :destroy Remove the account record instead of suspending + # @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts + # @option [Boolean] :reserve_username Keep account record + # @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads + # @option [Time] :suspended_at Only applicable when :reserve_username is true def call(account, **options) @account = account - @options = options + @options = { reserve_username: true, reserve_email: true }.merge(options) + + if @account.local? && @account.user_unconfirmed_or_pending? + @options[:reserve_email] = false + @options[:reserve_username] = false + @options[:skip_side_effects] = true + end reject_follows! purge_user! @@ -60,27 +71,39 @@ class SuspendAccountService < BaseService def purge_user! return if !@account.local? || @account.user.nil? - if @options[:including_user] - @options[:destroy] = true if !@account.user_confirmed? || @account.user_pending? - @account.user.destroy - else + if @options[:reserve_email] @account.user.disable! @account.user.invites.where(uses: 0).destroy_all + else + @account.user.destroy end end def purge_content! - distribute_delete_actor! if @account.local? && !@options[:skip_distribution] + distribute_delete_actor! if @account.local? && !@options[:skip_side_effects] @account.statuses.reorder(nil).find_in_batches do |statuses| - BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy]) + statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username] + BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects]) + end + + @account.media_attachments.reorder(nil).find_each do |media_attachment| + next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id) + + media_attachment.destroy + end + + @account.polls.reorder(nil).find_each do |poll| + next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id) + + poll.destroy end associations_for_destruction.each do |association_name| destroy_all(@account.public_send(association_name)) end - @account.destroy if @options[:destroy] + @account.destroy unless @options[:reserve_username] end def purge_profile! @@ -88,11 +111,13 @@ class SuspendAccountService < BaseService # there is no point wasting time updating # its values first - return if @options[:destroy] + return unless @options[:reserve_username] @account.silenced_at = nil @account.suspended_at = @options[:suspended_at] || Time.now.utc @account.locked = false + @account.memorial = false + @account.discoverable = false @account.display_name = '' @account.note = '' @account.fields = [] @@ -100,6 +125,7 @@ class SuspendAccountService < BaseService @account.followers_count = 0 @account.following_count = 0 @account.moved_to_account = nil + @account.trust_level = :untrusted @account.avatar.destroy @account.header.destroy @account.save! @@ -135,11 +161,15 @@ class SuspendAccountService < BaseService Account.inboxes - delivery_inboxes end + def reported_status_ids + @reported_status_ids ||= Report.where(target_account: @account).unresolved.pluck(:status_ids).flatten.uniq + end + def associations_for_destruction - if @options[:destroy] - ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY - else + if @options[:reserve_username] ASSOCIATIONS_ON_SUSPEND + else + ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY end end end diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb index d4387c1a1..bd1ad328d 100644 --- a/app/services/unallow_domain_service.rb +++ b/app/services/unallow_domain_service.rb @@ -3,7 +3,7 @@ class UnallowDomainService < BaseService def call(domain_allow) Account.where(domain: domain_allow.domain).find_each do |account| - SuspendAccountService.new.call(account, destroy: true) + SuspendAccountService.new.call(account, reserve_username: false) end domain_allow.destroy diff --git a/app/workers/admin/suspension_worker.rb b/app/workers/admin/suspension_worker.rb index ae8b24d8c..83c815efd 100644 --- a/app/workers/admin/suspension_worker.rb +++ b/app/workers/admin/suspension_worker.rb @@ -6,6 +6,6 @@ class Admin::SuspensionWorker sidekiq_options queue: 'pull' def perform(account_id, remove_user = false) - SuspendAccountService.new.call(Account.find(account_id), including_user: remove_user) + SuspendAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: !remove_user) end end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index b16bf2e38..a09a6ab04 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -185,7 +185,7 @@ module Mastodon end say("Deleting user with #{account.statuses_count} statuses, this might take a while...") - SuspendAccountService.new.call(account, including_user: true) + SuspendAccountService.new.call(account, reserve_email: false) say('OK', :green) end @@ -239,7 +239,7 @@ module Mastodon end if [404, 410].include?(code) - SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + SuspendAccountService.new.call(account, reserve_username: false) unless options[:dry_run] 1 else # Touch account even during dry run to avoid getting the account into the window again diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index c612c2d72..8e52de1c3 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -42,7 +42,7 @@ module Mastodon end processed, = parallelize_with_progress(scope) do |account| - SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run] end DomainBlock.where(domain: domain).destroy_all unless options[:dry_run] diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb index bd146b795..2a1598123 100644 --- a/spec/controllers/admin/reported_statuses_controller_spec.rb +++ b/spec/controllers/admin/reported_statuses_controller_spec.rb @@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true) end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 6b06343ef..d9690d83f 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -65,7 +65,7 @@ describe Admin::StatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true) end end diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb index f9c58c90f..68d84a737 100644 --- a/spec/models/form/status_batch_spec.rb +++ b/spec/models/form/status_batch_spec.rb @@ -41,12 +41,12 @@ describe Form::StatusBatch do it 'call RemovalWorker' do form.save - expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status.id, immediate: true) end it 'do not call RemovalWorker' do form.save - expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false) + expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, immediate: true) end end end -- cgit From e8703c281e5bf6538f620af1d0b24fdef7de7fc2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 13 Sep 2019 19:15:47 +0200 Subject: Fix web push subscriptions being deleted on rate limit or timeout (#11826) --- app/workers/web/push_notification_worker.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/web/push_notification_worker.rb b/app/workers/web/push_notification_worker.rb index 901043975..46aeaa30b 100644 --- a/app/workers/web/push_notification_worker.rb +++ b/app/workers/web/push_notification_worker.rb @@ -11,7 +11,13 @@ class Web::PushNotificationWorker subscription.push(notification) unless notification.activity.nil? rescue Webpush::ResponseError => e - subscription.destroy! if (400..499).cover?(e.response.code.to_i) + code = e.response.code.to_i + + if (400..499).cover?(code) && ![408, 429].include?(code) + subscription.destroy! + else + raise e + end rescue ActiveRecord::RecordNotFound true end -- cgit From c21386cff5ff6d86d8887e4a5dde1cf910ab84a0 Mon Sep 17 00:00:00 2001 From: abcang Date: Tue, 17 Sep 2019 15:44:25 +0900 Subject: Improve IP cleanup query (#11871) --- app/workers/scheduler/ip_cleanup_scheduler.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/workers') diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb index 42620332e..4f44078d8 100644 --- a/app/workers/scheduler/ip_cleanup_scheduler.rb +++ b/app/workers/scheduler/ip_cleanup_scheduler.rb @@ -9,7 +9,7 @@ class Scheduler::IpCleanupScheduler def perform time_ago = RETENTION_PERIOD.ago - SessionActivation.where('updated_at < ?', time_ago).destroy_all - User.where('last_sign_in_at < ?', time_ago).update_all(last_sign_in_ip: nil) + SessionActivation.where('updated_at < ?', time_ago).in_batches.destroy_all + User.where('last_sign_in_at < ?', time_ago).where.not(last_sign_in_ip: nil).in_batches.update_all(last_sign_in_ip: nil) end end -- cgit From 38dc51b2d68e3e03f429419f7318e868fa24c49d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 17 Sep 2019 08:44:45 +0200 Subject: Fix Move handler queuing jobs that will fail if account is suspended (#11864) Don't put Move handler on cooldown if it didn't run. Skip unmerging from timelines to save unnecessary work. --- app/lib/activitypub/activity/move.rb | 11 +++++++++-- app/services/unfollow_service.rb | 11 +++++++++-- app/workers/unfollow_follow_worker.rb | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) (limited to 'app/workers') diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index d7a5f595c..6c6a2b967 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -10,10 +10,13 @@ class ActivityPub::Activity::Move < ActivityPub::Activity target_account = ActivityPub::FetchRemoteAccountService.new.call(target_uri) - return if target_account.nil? || !target_account.also_known_as.include?(origin_account.uri) + if target_account.nil? || target_account.suspended? || !target_account.also_known_as.include?(origin_account.uri) + unmark_as_processing! + return + end # In case for some reason we didn't have a redirect for the profile already, set it - origin_account.update(moved_to_account: target_account) if origin_account.moved_to_account_id.nil? + origin_account.update(moved_to_account: target_account) # Initiate a re-follow for each follower origin_account.followers.local.select(:id).find_in_batches do |follower_accounts| @@ -40,4 +43,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity def mark_as_processing! redis.setex("move_in_progress:#{@account.id}", PROCESSING_COOLDOWN, true) end + + def unmark_as_processing! + redis.del("move_in_progress:#{@account.id}") + end end diff --git a/app/services/unfollow_service.rb b/app/services/unfollow_service.rb index b7033d7eb..151f3674f 100644 --- a/app/services/unfollow_service.rb +++ b/app/services/unfollow_service.rb @@ -6,9 +6,12 @@ class UnfollowService < BaseService # Unfollow and notify the remote user # @param [Account] source_account Where to unfollow from # @param [Account] target_account Which to unfollow - def call(source_account, target_account) + # @param [Hash] options + # @option [Boolean] :skip_unmerge + def call(source_account, target_account, options = {}) @source_account = source_account @target_account = target_account + @options = options unfollow! || undo_follow_request! end @@ -21,9 +24,11 @@ class UnfollowService < BaseService return unless follow follow.destroy! + create_notification(follow) if !@target_account.local? && @target_account.activitypub? create_reject_notification(follow) if @target_account.local? && !@source_account.local? && @source_account.activitypub? - UnmergeWorker.perform_async(@target_account.id, @source_account.id) + UnmergeWorker.perform_async(@target_account.id, @source_account.id) unless @options[:skip_unmerge] + follow end @@ -33,7 +38,9 @@ class UnfollowService < BaseService return unless follow_request follow_request.destroy! + create_notification(follow_request) unless @target_account.local? + follow_request end diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb index 50d3bf034..95549e107 100644 --- a/app/workers/unfollow_follow_worker.rb +++ b/app/workers/unfollow_follow_worker.rb @@ -11,7 +11,7 @@ class UnfollowFollowWorker new_target_account = Account.find(new_target_account_id) FollowService.new.call(follower_account, new_target_account) - UnfollowService.new.call(follower_account, old_target_account) + UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true) rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError true end -- cgit From 3ed94dcc1acf73f1d0d1ab43567b88ee953f57c9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 19 Sep 2019 20:58:19 +0200 Subject: Add account migration UI (#11846) Fix #10736 - Change data export to be available for non-functional accounts - Change non-functional accounts to include redirecting accounts --- .../concerns/export_controller_concern.rb | 7 ++ app/controllers/settings/aliases_controller.rb | 42 +++++++++++ app/controllers/settings/exports_controller.rb | 7 ++ app/controllers/settings/migrations_controller.rb | 48 ++++++++++--- app/helpers/settings_helper.rb | 8 +++ app/models/account_alias.rb | 41 +++++++++++ app/models/account_migration.rb | 74 +++++++++++++++++++ app/models/concerns/account_associations.rb | 2 + app/models/form/migration.rb | 25 ------- app/models/remote_follow.rb | 2 +- app/models/user.rb | 2 +- app/serializers/activitypub/move_serializer.rb | 26 +++++++ app/views/auth/registrations/_status.html.haml | 30 ++++---- app/views/auth/registrations/edit.html.haml | 2 +- app/views/settings/aliases/index.html.haml | 29 ++++++++ app/views/settings/exports/show.html.haml | 4 ++ app/views/settings/migrations/show.html.haml | 84 +++++++++++++++++++--- app/views/settings/profiles/show.html.haml | 5 ++ .../activitypub/move_distribution_worker.rb | 32 +++++++++ config/locales/en.yml | 38 ++++++++-- config/locales/simple_form.en.yml | 10 +++ config/navigation.rb | 8 +-- config/routes.rb | 8 ++- .../20190914202517_create_account_migrations.rb | 12 ++++ .../20190915194355_create_account_aliases.rb | 11 +++ db/schema.rb | 23 ++++++ .../settings/migrations_controller_spec.rb | 14 ++-- spec/fabricators/account_alias_fabricator.rb | 5 ++ spec/fabricators/account_migration_fabricator.rb | 6 ++ spec/models/account_alias_spec.rb | 5 ++ spec/models/account_migration_spec.rb | 5 ++ 31 files changed, 542 insertions(+), 73 deletions(-) create mode 100644 app/controllers/settings/aliases_controller.rb create mode 100644 app/models/account_alias.rb create mode 100644 app/models/account_migration.rb delete mode 100644 app/models/form/migration.rb create mode 100644 app/serializers/activitypub/move_serializer.rb create mode 100644 app/views/settings/aliases/index.html.haml create mode 100644 app/workers/activitypub/move_distribution_worker.rb create mode 100644 db/migrate/20190914202517_create_account_migrations.rb create mode 100644 db/migrate/20190915194355_create_account_aliases.rb create mode 100644 spec/fabricators/account_alias_fabricator.rb create mode 100644 spec/fabricators/account_migration_fabricator.rb create mode 100644 spec/models/account_alias_spec.rb create mode 100644 spec/models/account_migration_spec.rb (limited to 'app/workers') diff --git a/app/controllers/concerns/export_controller_concern.rb b/app/controllers/concerns/export_controller_concern.rb index e20b71a30..bfe990c82 100644 --- a/app/controllers/concerns/export_controller_concern.rb +++ b/app/controllers/concerns/export_controller_concern.rb @@ -5,7 +5,10 @@ module ExportControllerConcern included do before_action :authenticate_user! + before_action :require_not_suspended! before_action :load_export + + skip_before_action :require_functional! end private @@ -27,4 +30,8 @@ module ExportControllerConcern def export_filename "#{controller_name}.csv" end + + def require_not_suspended! + forbidden if current_account.suspended? + end end diff --git a/app/controllers/settings/aliases_controller.rb b/app/controllers/settings/aliases_controller.rb new file mode 100644 index 000000000..2b675f065 --- /dev/null +++ b/app/controllers/settings/aliases_controller.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class Settings::AliasesController < Settings::BaseController + layout 'admin' + + before_action :authenticate_user! + before_action :set_aliases, except: :destroy + before_action :set_alias, only: :destroy + + def index + @alias = current_account.aliases.build + end + + def create + @alias = current_account.aliases.build(resource_params) + + if @alias.save + redirect_to settings_aliases_path, notice: I18n.t('aliases.created_msg') + else + render :show + end + end + + def destroy + @alias.destroy! + redirect_to settings_aliases_path, notice: I18n.t('aliases.deleted_msg') + end + + private + + def resource_params + params.require(:account_alias).permit(:acct) + end + + def set_alias + @alias = current_account.aliases.find(params[:id]) + end + + def set_aliases + @aliases = current_account.aliases.order(id: :desc).reject(&:new_record?) + end +end diff --git a/app/controllers/settings/exports_controller.rb b/app/controllers/settings/exports_controller.rb index 3012fbf77..0e93d07a9 100644 --- a/app/controllers/settings/exports_controller.rb +++ b/app/controllers/settings/exports_controller.rb @@ -6,6 +6,9 @@ class Settings::ExportsController < Settings::BaseController layout 'admin' before_action :authenticate_user! + before_action :require_not_suspended! + + skip_before_action :require_functional! def show @export = Export.new(current_account) @@ -34,4 +37,8 @@ class Settings::ExportsController < Settings::BaseController def lock_options { redis: Redis.current, key: "backup:#{current_user.id}" } end + + def require_not_suspended! + forbidden if current_account.suspended? + end end diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb index 59eb48779..90092c692 100644 --- a/app/controllers/settings/migrations_controller.rb +++ b/app/controllers/settings/migrations_controller.rb @@ -4,31 +4,59 @@ class Settings::MigrationsController < Settings::BaseController layout 'admin' before_action :authenticate_user! + before_action :require_not_suspended! + before_action :set_migrations + before_action :set_cooldown + + skip_before_action :require_functional! def show - @migration = Form::Migration.new(account: current_account.moved_to_account) + @migration = current_account.migrations.build end - def update - @migration = Form::Migration.new(resource_params) + def create + @migration = current_account.migrations.build(resource_params) - if @migration.valid? && migration_account_changed? - current_account.update!(moved_to_account: @migration.account) + if @migration.save_with_challenge(current_user) + current_account.update!(moved_to_account: @migration.target_account) ActivityPub::UpdateDistributionWorker.perform_async(current_account.id) - redirect_to settings_migration_path, notice: I18n.t('migrations.updated_msg') + ActivityPub::MoveDistributionWorker.perform_async(@migration.id) + redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct) else render :show end end + def cancel + if current_account.moved_to_account_id.present? + current_account.update!(moved_to_account: nil) + ActivityPub::UpdateDistributionWorker.perform_async(current_account.id) + end + + redirect_to settings_migration_path, notice: I18n.t('migrations.cancelled_msg') + end + + helper_method :on_cooldown? + private def resource_params - params.require(:migration).permit(:acct) + params.require(:account_migration).permit(:acct, :current_password, :current_username) + end + + def set_migrations + @migrations = current_account.migrations.includes(:target_account).order(id: :desc).reject(&:new_record?) + end + + def set_cooldown + @cooldown = current_account.migrations.within_cooldown.first + end + + def on_cooldown? + @cooldown.present? end - def migration_account_changed? - current_account.moved_to_account_id != @migration.account&.id && - current_account.id != @migration.account&.id + def require_not_suspended! + forbidden if current_account.suspended? end end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 2b3fd1263..ecc73baf5 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -87,4 +87,12 @@ module SettingsHelper 'desktop' end end + + def compact_account_link_to(account) + return if account.nil? + + link_to ActivityPub::TagManager.instance.url_for(account), class: 'name-tag', title: account.acct do + safe_join([image_tag(account.avatar.url, width: 15, height: 15, alt: display_name(account), class: 'avatar'), content_tag(:span, account.acct, class: 'username')], ' ') + end + end end diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb new file mode 100644 index 000000000..e9a0dd79e --- /dev/null +++ b/app/models/account_alias.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: account_aliases +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) +# acct :string default(""), not null +# uri :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class AccountAlias < ApplicationRecord + belongs_to :account + + validates :acct, presence: true, domain: { acct: true } + validates :uri, presence: true + + before_validation :set_uri + after_create :add_to_account + after_destroy :remove_from_account + + private + + def set_uri + target_account = ResolveAccountService.new.call(acct) + self.uri = ActivityPub::TagManager.instance.uri_for(target_account) unless target_account.nil? + rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + # Validation will take care of it + end + + def add_to_account + account.update(also_known_as: account.also_known_as + [uri]) + end + + def remove_from_account + account.update(also_known_as: account.also_known_as.reject { |x| x == uri }) + end +end diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb new file mode 100644 index 000000000..15830bffb --- /dev/null +++ b/app/models/account_migration.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: account_migrations +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) +# acct :string default(""), not null +# followers_count :bigint(8) default(0), not null +# target_account_id :bigint(8) +# created_at :datetime not null +# updated_at :datetime not null +# + +class AccountMigration < ApplicationRecord + COOLDOWN_PERIOD = 30.days.freeze + + belongs_to :account + belongs_to :target_account, class_name: 'Account' + + before_validation :set_target_account + before_validation :set_followers_count + + validates :acct, presence: true, domain: { acct: true } + validate :validate_migration_cooldown + validate :validate_target_account + + scope :within_cooldown, ->(now = Time.now.utc) { where(arel_table[:created_at].gteq(now - COOLDOWN_PERIOD)) } + + attr_accessor :current_password, :current_username + + def save_with_challenge(current_user) + if current_user.encrypted_password.present? + errors.add(:current_password, :invalid) unless current_user.valid_password?(current_password) + else + errors.add(:current_username, :invalid) unless account.username == current_username + end + + return false unless errors.empty? + + save + end + + def cooldown_at + created_at + COOLDOWN_PERIOD + end + + private + + def set_target_account + self.target_account = ResolveAccountService.new.call(acct) + rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + # Validation will take care of it + end + + def set_followers_count + self.followers_count = account.followers_count + end + + def validate_target_account + if target_account.nil? + errors.add(:acct, I18n.t('migrations.errors.not_found')) + else + errors.add(:acct, I18n.t('migrations.errors.missing_also_known_as')) unless target_account.also_known_as.include?(ActivityPub::TagManager.instance.uri_for(account)) + errors.add(:acct, I18n.t('migrations.errors.already_moved')) if account.moved_to_account_id.present? && account.moved_to_account_id == target_account.id + errors.add(:acct, I18n.t('migrations.errors.move_to_self')) if account.id == target_account.id + end + end + + def validate_migration_cooldown + errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists? + end +end diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index 1db7771c7..c9cc5c610 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -52,6 +52,8 @@ module AccountAssociations # Account migrations belongs_to :moved_to_account, class_name: 'Account', optional: true + has_many :migrations, class_name: 'AccountMigration', dependent: :destroy, inverse_of: :account + has_many :aliases, class_name: 'AccountAlias', dependent: :destroy, inverse_of: :account # Hashtags has_and_belongs_to_many :tags diff --git a/app/models/form/migration.rb b/app/models/form/migration.rb deleted file mode 100644 index c2a8655e1..000000000 --- a/app/models/form/migration.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -class Form::Migration - include ActiveModel::Validations - - attr_accessor :acct, :account - - def initialize(attrs = {}) - @account = attrs[:account] - @acct = attrs[:account].acct unless @account.nil? - @acct = attrs[:acct].gsub(/\A@/, '').strip unless attrs[:acct].nil? - end - - def valid? - return false unless super - set_account - errors.empty? - end - - private - - def set_account - self.account = (ResolveAccountService.new.call(acct) if account.nil? && acct.present?) - end -end diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb index 52dd3f67b..5ea535287 100644 --- a/app/models/remote_follow.rb +++ b/app/models/remote_follow.rb @@ -49,7 +49,7 @@ class RemoteFollow end def fetch_template! - return missing_resource if acct.blank? + return missing_resource_error if acct.blank? _, domain = acct.split('@') diff --git a/app/models/user.rb b/app/models/user.rb index b48455802..9a19a53b3 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -168,7 +168,7 @@ class User < ApplicationRecord end def functional? - confirmed? && approved? && !disabled? && !account.suspended? + confirmed? && approved? && !disabled? && !account.suspended? && account.moved_to_account_id.nil? end def unconfirmed_or_pending? diff --git a/app/serializers/activitypub/move_serializer.rb b/app/serializers/activitypub/move_serializer.rb new file mode 100644 index 000000000..5675875fa --- /dev/null +++ b/app/serializers/activitypub/move_serializer.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class ActivityPub::MoveSerializer < ActivityPub::Serializer + attributes :id, :type, :target, :actor + attribute :virtual_object, key: :object + + def id + [ActivityPub::TagManager.instance.uri_for(object.account), '#moves/', object.id].join + end + + def type + 'Move' + end + + def target + ActivityPub::TagManager.instance.uri_for(object.target_account) + end + + def virtual_object + ActivityPub::TagManager.instance.uri_for(object.account) + end + + def actor + ActivityPub::TagManager.instance.uri_for(object.account) + end +end diff --git a/app/views/auth/registrations/_status.html.haml b/app/views/auth/registrations/_status.html.haml index b38a83d67..47112dae0 100644 --- a/app/views/auth/registrations/_status.html.haml +++ b/app/views/auth/registrations/_status.html.haml @@ -1,16 +1,22 @@ %h3= t('auth.status.account_status') -- if @user.account.suspended? - %span.negative-hint= t('user_mailer.warning.explanation.suspend') -- elsif @user.disabled? - %span.negative-hint= t('user_mailer.warning.explanation.disable') -- elsif @user.account.silenced? - %span.warning-hint= t('user_mailer.warning.explanation.silence') -- elsif !@user.confirmed? - %span.warning-hint= t('auth.status.confirming') -- elsif !@user.approved? - %span.warning-hint= t('auth.status.pending') -- else - %span.positive-hint= t('auth.status.functional') +.simple_form + %p.hint + - if @user.account.suspended? + %span.negative-hint= t('user_mailer.warning.explanation.suspend') + - elsif @user.disabled? + %span.negative-hint= t('user_mailer.warning.explanation.disable') + - elsif @user.account.silenced? + %span.warning-hint= t('user_mailer.warning.explanation.silence') + - elsif !@user.confirmed? + %span.warning-hint= t('auth.status.confirming') + = link_to t('auth.didnt_get_confirmation'), new_user_confirmation_path + - elsif !@user.approved? + %span.warning-hint= t('auth.status.pending') + - elsif @user.account.moved_to_account_id.present? + %span.positive-hint= t('auth.status.redirecting_to', acct: @user.account.moved_to_account.acct) + = link_to t('migrations.cancel'), settings_migration_path + - else + %span.positive-hint= t('auth.status.functional') %hr.spacer/ diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml index 710ee5c68..885171c58 100644 --- a/app/views/auth/registrations/edit.html.haml +++ b/app/views/auth/registrations/edit.html.haml @@ -13,7 +13,7 @@ .fields-row__column.fields-group.fields-row__column-6 = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, disabled: current_account.suspended? .fields-row__column.fields-group.fields-row__column-6 - = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended? + = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?, hint: false .fields-row .fields-row__column.fields-group.fields-row__column-6 diff --git a/app/views/settings/aliases/index.html.haml b/app/views/settings/aliases/index.html.haml new file mode 100644 index 000000000..5b6986368 --- /dev/null +++ b/app/views/settings/aliases/index.html.haml @@ -0,0 +1,29 @@ +- content_for :page_title do + = t('settings.aliases') + += simple_form_for @alias, url: settings_aliases_path do |f| + = render 'shared/error_messages', object: @alias + + %p.hint= t('aliases.hint_html') + + %hr.spacer/ + + .fields-group + = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' } + + .actions + = f.button :button, t('aliases.add_new'), type: :submit, class: 'button' + +%hr.spacer/ + +.table-wrapper + %table.table.inline-table + %thead + %tr + %th= t('simple_form.labels.account_alias.acct') + %th + %tbody + - @aliases.each do |account_alias| + %tr + %td= account_alias.acct + %td= table_link_to 'trash', t('aliases.remove'), settings_alias_path(account_alias), data: { method: :delete } diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index b13cea976..76ff76bd9 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -37,12 +37,16 @@ %td= number_with_delimiter @export.total_domain_blocks %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv) +%hr.spacer/ + %p.muted-hint= t('exports.archive_takeout.hint_html') - if policy(:backup).create? %p= link_to t('exports.archive_takeout.request'), settings_export_path, class: 'button', method: :post - unless @backups.empty? + %hr.spacer/ + .table-wrapper %table.table %thead diff --git a/app/views/settings/migrations/show.html.haml b/app/views/settings/migrations/show.html.haml index c69061d50..1e5c47726 100644 --- a/app/views/settings/migrations/show.html.haml +++ b/app/views/settings/migrations/show.html.haml @@ -1,17 +1,85 @@ - content_for :page_title do = t('settings.migrate') -= simple_form_for @migration, as: :migration, url: settings_migration_path, html: { method: :put } do |f| - - if @migration.account - %p.hint= t('migrations.currently_redirecting') +.simple_form + - if current_account.moved_to_account.present? + .fields-row + .fields-row__column.fields-group.fields-row__column-6 + = render 'application/card', account: current_account.moved_to_account + .fields-row__column.fields-group.fields-row__column-6 + %p.hint + %span.positive-hint= t('migrations.redirecting_to', acct: current_account.moved_to_account.acct) - .fields-group - = render partial: 'application/card', locals: { account: @migration.account } + %p.hint= t('migrations.cancel_explanation') + + %p.hint= link_to t('migrations.cancel'), cancel_settings_migration_path, data: { method: :post } + - else + %p.hint + %span.positive-hint= t('migrations.not_redirecting') + +%hr.spacer/ + +%h3= t 'migrations.proceed_with_move' + += simple_form_for @migration, url: settings_migration_path do |f| + - if on_cooldown? + %span.warning-hint= t('migrations.on_cooldown', count: ((@cooldown.cooldown_at - Time.now.utc) / 1.day.seconds).ceil) + - else + %p.hint= t('migrations.warning.before') + + %ul.hint + %li.warning-hint= t('migrations.warning.followers') + %li.warning-hint= t('migrations.warning.other_data') + %li.warning-hint= t('migrations.warning.backreference_required') + %li.warning-hint= t('migrations.warning.cooldown') + %li.warning-hint= t('migrations.warning.disabled_account') + + %hr.spacer/ = render 'shared/error_messages', object: @migration - .fields-group - = f.input :acct, placeholder: t('migrations.acct') + .fields-row + .fields-row__column.fields-group.fields-row__column-6 + = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }, disabled: on_cooldown? + + .fields-row__column.fields-group.fields-row__column-6 + - if current_user.encrypted_password.present? + = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown? + - else + = f.input :current_username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown? .actions - = f.button :button, t('migrations.proceed'), type: :submit, class: 'negative' + = f.button :button, t('migrations.proceed_with_move'), type: :submit, class: 'button button--destructive', disabled: on_cooldown? + +- unless @migrations.empty? + %hr.spacer/ + + %h3= t 'migrations.past_migrations' + + %hr.spacer/ + + .table-wrapper + %table.table.inline-table + %thead + %tr + %th= t('migrations.acct') + %th= t('migrations.followers_count') + %th + %tbody + - @migrations.each do |migration| + %tr + %td + - if migration.target_account.present? + = compact_account_link_to migration.target_account + - else + = migration.acct + + %td= number_with_delimiter migration.followers_count + + %td + %time.time-ago{ datetime: migration.created_at.iso8601, title: l(migration.created_at) }= l(migration.created_at) + +%hr.spacer/ + +%h3= t 'migrations.incoming_migrations' +%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path) diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml index f042011d6..6929f54f3 100644 --- a/app/views/settings/profiles/show.html.haml +++ b/app/views/settings/profiles/show.html.haml @@ -60,6 +60,11 @@ %h6= t('auth.migrate_account') %p.muted-hint= t('auth.migrate_account_html', path: settings_migration_path) +%hr.spacer/ + +%h6= t 'migrations.incoming_migrations' +%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path) + - if open_deletion? %hr.spacer/ diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb new file mode 100644 index 000000000..396d5258f --- /dev/null +++ b/app/workers/activitypub/move_distribution_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class ActivityPub::MoveDistributionWorker + include Sidekiq::Worker + include Payloadable + + sidekiq_options queue: 'push' + + def perform(migration_id) + @migration = AccountMigration.find(migration_id) + + ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + [signed_payload, @account.id, inbox_url] + end + + ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url| + [signed_payload, @account.id, inbox_url] + end + rescue ActiveRecord::RecordNotFound + true + end + + private + + def inboxes + @inboxes ||= @migration.account.followers.inboxes + end + + def signed_payload + @signed_payload ||= Oj.dump(serialize_payload(@migration, ActivityPub::MoveSerializer, signer: @account)) + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index dabb679e7..c29c7f871 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -554,6 +554,12 @@ en: new_trending_tag: body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.' subject: New hashtag up for review on %{instance} (#%{name}) + aliases: + add_new: Create alias + created_msg: Successfully created a new alias. You can now initiate the move from the old account. + deleted_msg: Successfully remove the alias. Moving from that account to this one will no longer be possible. + hint_html: If you want to move from another account to this one, here you can create an alias, which is required before you can proceed with moving followers from the old account to this one. This action by itself is harmless and reversible. The account migration is initiated from the old account. + remove: Unlink alias appearance: advanced_web_interface: Advanced web interface advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.' @@ -613,6 +619,7 @@ en: confirming: Waiting for e-mail confirmation to be completed. functional: Your account is fully operational. pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved. + redirecting_to: Your account is inactive because it is currently redirecting to %{acct}. trouble_logging_in: Trouble logging in? authorize_follow: already_following: You are already following this account @@ -801,10 +808,32 @@ en: images_and_video: Cannot attach a video to a status that already contains images too_many: Cannot attach more than 4 files migrations: - acct: username@domain of the new account - currently_redirecting: 'Your profile is set to redirect to:' - proceed: Save - updated_msg: Your account migration setting successfully updated! + acct: Moved to + cancel: Cancel redirect + cancel_explanation: Cancelling the redirect will re-activate your current account, but will not bring back followers that have been moved to that account. + cancelled_msg: Successfully cancelled the redirect. + errors: + already_moved: is the same account you have already moved to + missing_also_known_as: is not back-referencing this account + move_to_self: cannot be current account + not_found: could not be found + on_cooldown: You are on cooldown + followers_count: Followers at time of move + incoming_migrations: Moving from a different account + incoming_migrations_html: To move from another account to this one, first you need to create an account alias. + moved_msg: Your account is now redirecting to %{acct} and your followers are being moved over. + not_redirecting: Your account is not redirecting to any other account currently. + on_cooldown: You have recently migrated your account. This function will become available again in %{count} days. + past_migrations: Past migrations + proceed_with_move: Move followers + redirecting_to: Your account is redirecting to %{acct}. + warning: + backreference_required: The new account must first be configured to back-reference this one + before: 'Before proceeding, please read these notes carefully:' + cooldown: After moving there is a cooldown period during which you will not be able to move again + disabled_account: Your current account will not be fully usable afterwards. However, you will have access to data export as well as re-activation. + followers: This action will move all followers from the current account to the new account + other_data: No other data will be moved automatically moderation: title: Moderation notification_mailer: @@ -950,6 +979,7 @@ en: settings: account: Account account_settings: Account settings + aliases: Account aliases appearance: Appearance authorized_apps: Authorized apps back: Back to Mastodon diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index c9ffcfc13..3d909e999 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -2,6 +2,10 @@ en: simple_form: hints: + account_alias: + acct: Specify the username@domain of the account you want to move from + account_migration: + acct: Specify the username@domain of the account you want to move to account_warning_preset: text: You can use toot syntax, such as URLs, hashtags and mentions admin_account_action: @@ -15,6 +19,8 @@ en: avatar: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px bot: This account mainly performs automated actions and might not be monitored context: One or multiple contexts where the filter should apply + current_password: For security purposes please enter the password of the current account + current_username: To confirm, please enter the username of the current account digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence discoverable: The profile directory is another way by which your account can reach a wider audience email: You will be sent a confirmation e-mail @@ -60,6 +66,10 @@ en: fields: name: Label value: Content + account_alias: + acct: Handle of the old account + account_migration: + acct: Handle of the new account account_warning_preset: text: Preset text admin_account_action: diff --git a/config/navigation.rb b/config/navigation.rb index 38668bbf7..32c299143 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -5,7 +5,7 @@ SimpleNavigation::Configuration.run do |navigation| n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_url n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url, if: -> { current_user.functional? } do |s| - s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url, highlights_on: %r{/settings/profile|/settings/migration} + s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url s.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_url s.item :identity_proofs, safe_join([fa_icon('key fw'), t('settings.identity_proofs')]), settings_identity_proofs_path, highlights_on: %r{/settings/identity_proofs*}, if: proc { current_account.identity_proofs.exists? } end @@ -20,13 +20,13 @@ SimpleNavigation::Configuration.run do |navigation| n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? } n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s| - s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete} + s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases} s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_url, highlights_on: %r{/settings/two_factor_authentication} s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_url end - n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url, if: -> { current_user.functional? } do |s| - s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url + n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url do |s| + s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url, if: -> { current_user.functional? } s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_url end diff --git a/config/routes.rb b/config/routes.rb index dcfa079a0..37e0cbdee 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -134,8 +134,14 @@ Rails.application.routes.draw do end resource :delete, only: [:show, :destroy] - resource :migration, only: [:show, :update] + resource :migration, only: [:show, :create] do + collection do + post :cancel + end + end + + resources :aliases, only: [:index, :create, :destroy] resources :sessions, only: [:destroy] resources :featured_tags, only: [:index, :create, :destroy] end diff --git a/db/migrate/20190914202517_create_account_migrations.rb b/db/migrate/20190914202517_create_account_migrations.rb new file mode 100644 index 000000000..cb9d71c09 --- /dev/null +++ b/db/migrate/20190914202517_create_account_migrations.rb @@ -0,0 +1,12 @@ +class CreateAccountMigrations < ActiveRecord::Migration[5.2] + def change + create_table :account_migrations do |t| + t.belongs_to :account, foreign_key: { on_delete: :cascade } + t.string :acct, null: false, default: '' + t.bigint :followers_count, null: false, default: 0 + t.belongs_to :target_account, foreign_key: { to_table: :accounts, on_delete: :nullify } + + t.timestamps + end + end +end diff --git a/db/migrate/20190915194355_create_account_aliases.rb b/db/migrate/20190915194355_create_account_aliases.rb new file mode 100644 index 000000000..32ce031d9 --- /dev/null +++ b/db/migrate/20190915194355_create_account_aliases.rb @@ -0,0 +1,11 @@ +class CreateAccountAliases < ActiveRecord::Migration[5.2] + def change + create_table :account_aliases do |t| + t.belongs_to :account, foreign_key: { on_delete: :cascade } + t.string :acct, null: false, default: '' + t.string :uri, null: false, default: '' + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 749f79dee..fabeb16f3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -15,6 +15,15 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "account_aliases", force: :cascade do |t| + t.bigint "account_id" + t.string "acct", default: "", null: false + t.string "uri", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_account_aliases_on_account_id" + end + create_table "account_conversations", force: :cascade do |t| t.bigint "account_id" t.bigint "conversation_id" @@ -49,6 +58,17 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do t.index ["account_id"], name: "index_account_identity_proofs_on_account_id" end + create_table "account_migrations", force: :cascade do |t| + t.bigint "account_id" + t.string "acct", default: "", null: false + t.bigint "followers_count", default: 0, null: false + t.bigint "target_account_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_account_migrations_on_account_id" + t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id" + end + create_table "account_moderation_notes", force: :cascade do |t| t.text "content", null: false t.bigint "account_id", null: false @@ -768,10 +788,13 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do t.index ["user_id"], name: "index_web_settings_on_user_id", unique: true end + add_foreign_key "account_aliases", "accounts", on_delete: :cascade add_foreign_key "account_conversations", "accounts", on_delete: :cascade add_foreign_key "account_conversations", "conversations", on_delete: :cascade add_foreign_key "account_domain_blocks", "accounts", name: "fk_206c6029bd", on_delete: :cascade add_foreign_key "account_identity_proofs", "accounts", on_delete: :cascade + add_foreign_key "account_migrations", "accounts", column: "target_account_id", on_delete: :nullify + add_foreign_key "account_migrations", "accounts", on_delete: :cascade add_foreign_key "account_moderation_notes", "accounts" add_foreign_key "account_moderation_notes", "accounts", column: "target_account_id" add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade diff --git a/spec/controllers/settings/migrations_controller_spec.rb b/spec/controllers/settings/migrations_controller_spec.rb index 4d814a45e..36e4ba86e 100644 --- a/spec/controllers/settings/migrations_controller_spec.rb +++ b/spec/controllers/settings/migrations_controller_spec.rb @@ -21,6 +21,7 @@ describe Settings::MigrationsController do let(:user) { Fabricate(:user, account: account) } let(:account) { Fabricate(:account, moved_to_account: moved_to_account) } + before { sign_in user, scope: :user } context 'when user does not have moved to account' do @@ -32,7 +33,7 @@ describe Settings::MigrationsController do end end - context 'when user does not have moved to account' do + context 'when user has a moved to account' do let(:moved_to_account) { Fabricate(:account) } it 'renders show page' do @@ -43,21 +44,22 @@ describe Settings::MigrationsController do end end - describe 'PUT #update' do + describe 'POST #create' do context 'when user is not sign in' do - subject { put :update } + subject { post :create } it_behaves_like 'authenticate user' end context 'when user is sign in' do - subject { put :update, params: { migration: { acct: acct } } } + subject { post :create, params: { account_migration: { acct: acct, current_password: '12345678' } } } + + let(:user) { Fabricate(:user, password: '12345678') } - let(:user) { Fabricate(:user) } before { sign_in user, scope: :user } context 'when migration account is changed' do - let(:acct) { Fabricate(:account) } + let(:acct) { Fabricate(:account, also_known_as: [ActivityPub::TagManager.instance.uri_for(user.account)]) } it 'updates moved to account' do is_expected.to redirect_to settings_migration_path diff --git a/spec/fabricators/account_alias_fabricator.rb b/spec/fabricators/account_alias_fabricator.rb new file mode 100644 index 000000000..94dde9bb8 --- /dev/null +++ b/spec/fabricators/account_alias_fabricator.rb @@ -0,0 +1,5 @@ +Fabricator(:account_alias) do + account + acct 'test@example.com' + uri 'https://example.com/users/test' +end diff --git a/spec/fabricators/account_migration_fabricator.rb b/spec/fabricators/account_migration_fabricator.rb new file mode 100644 index 000000000..3b3fc2077 --- /dev/null +++ b/spec/fabricators/account_migration_fabricator.rb @@ -0,0 +1,6 @@ +Fabricator(:account_migration) do + account + target_account + followers_count 1234 + acct 'test@example.com' +end diff --git a/spec/models/account_alias_spec.rb b/spec/models/account_alias_spec.rb new file mode 100644 index 000000000..27ec215aa --- /dev/null +++ b/spec/models/account_alias_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe AccountAlias, type: :model do + +end diff --git a/spec/models/account_migration_spec.rb b/spec/models/account_migration_spec.rb new file mode 100644 index 000000000..8461b4b28 --- /dev/null +++ b/spec/models/account_migration_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe AccountMigration, type: :model do + +end -- cgit From a90243a712ac17708de898e302e337de05b7699d Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Sat, 21 Sep 2019 16:11:58 +0900 Subject: Fixed missing account in MoveDistributionWorker (#11906) --- app/workers/activitypub/move_distribution_worker.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/workers') diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb index 396d5258f..bf1c0e7ae 100644 --- a/app/workers/activitypub/move_distribution_worker.rb +++ b/app/workers/activitypub/move_distribution_worker.rb @@ -8,6 +8,7 @@ class ActivityPub::MoveDistributionWorker def perform(migration_id) @migration = AccountMigration.find(migration_id) + @account = @migration.account ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| [signed_payload, @account.id, inbox_url] -- cgit From 368a87755b4b12c37deb415e10e03c709012f698 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 29 Sep 2019 16:23:13 +0200 Subject: Fix account migration not affecting followers on origin server (#11980) --- app/controllers/settings/migrations_controller.rb | 4 +-- app/lib/activitypub/activity/move.rb | 6 +---- app/services/move_service.rb | 32 ++++++++++++++++++++++ app/workers/move_worker.rb | 33 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 app/services/move_service.rb create mode 100644 app/workers/move_worker.rb (limited to 'app/workers') diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb index 00bde1d61..68304bb51 100644 --- a/app/controllers/settings/migrations_controller.rb +++ b/app/controllers/settings/migrations_controller.rb @@ -18,9 +18,7 @@ class Settings::MigrationsController < Settings::BaseController @migration = current_account.migrations.build(resource_params) if @migration.save_with_challenge(current_user) - current_account.update!(moved_to_account: @migration.target_account) - ActivityPub::UpdateDistributionWorker.perform_async(current_account.id) - ActivityPub::MoveDistributionWorker.perform_async(@migration.id) + MoveService.new.call(@migration) redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct) else render :show diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 6c6a2b967..12bb82d25 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -19,11 +19,7 @@ class ActivityPub::Activity::Move < ActivityPub::Activity origin_account.update(moved_to_account: target_account) # Initiate a re-follow for each follower - origin_account.followers.local.select(:id).find_in_batches do |follower_accounts| - UnfollowFollowWorker.push_bulk(follower_accounts.map(&:id)) do |follower_account_id| - [follower_account_id, origin_account.id, target_account.id] - end - end + MoveWorker.perform_async(origin_account.id, target_account.id) end private diff --git a/app/services/move_service.rb b/app/services/move_service.rb new file mode 100644 index 000000000..da0c62c4e --- /dev/null +++ b/app/services/move_service.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class MoveService < BaseService + def call(migration) + @migration = migration + @source_account = migration.account + @target_account = migration.target_account + + update_redirect! + process_local_relationships! + distribute_update! + distribute_move! + end + + private + + def update_redirect! + @source_account.update!(moved_to_account: @target_account) + end + + def process_local_relationships! + MoveWorker.perform_async(@source_account.id, @target_account.id) + end + + def distribute_update! + ActivityPub::UpdateDistributionWorker.perform_async(@source_account.id) + end + + def distribute_move! + ActivityPub::MoveDistributionWorker.perform_async(@migration.id) + end +end diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb new file mode 100644 index 000000000..22788716f --- /dev/null +++ b/app/workers/move_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class MoveWorker + include Sidekiq::Worker + + def perform(source_account_id, target_account_id) + @source_account = Account.find(source_account_id) + @target_account = Account.find(target_account_id) + + if @target_account.local? + rewrite_follows! + else + queue_follow_unfollows! + end + rescue ActiveRecord::RecordNotFound + true + end + + private + + def rewrite_follows! + @source_account.passive_relationships + .where(account: Account.local) + .in_batches + .update_all(target_account: @target_account) + end + + def queue_follow_unfollows! + @source_account.followers.local.select(:id).find_in_batches do |accounts| + UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id] } + end + end +end -- cgit From 9a1be333b8f2a1bb2a25e77053efa16c3e30f068 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Oct 2019 17:50:48 +0200 Subject: Fix wrong attribute in MoveWorker (#12066) --- app/workers/move_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb index 22788716f..83f7090ee 100644 --- a/app/workers/move_worker.rb +++ b/app/workers/move_worker.rb @@ -22,7 +22,7 @@ class MoveWorker @source_account.passive_relationships .where(account: Account.local) .in_batches - .update_all(target_account: @target_account) + .update_all(target_account_id: @target_account.id) end def queue_follow_unfollows! -- cgit From 66c1fe0495b7e4728ad776e9c41a70c180640ea5 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 7 Nov 2019 08:05:07 +0100 Subject: Fix various issues with account migration (#12301) * Fix being able to follow oneself by moving to an account that was following the old one * Add specs * Add spec to catch MoveWorker issue with local followers following both accounts * Fix move worker breaking when a local account follows both source and target accounts * Fix migration from remote to local account not sending Undo Follow * Fix show_reblogs not being preserved for moved account's followers --- app/services/follow_service.rb | 4 +- app/workers/move_worker.rb | 8 +++- app/workers/unfollow_follow_worker.rb | 7 +++- spec/workers/move_worker_spec.rb | 63 +++++++++++++++++++++++++++++ spec/workers/unfollow_follow_worker_spec.rb | 50 +++++++++++++++++++++++ 5 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 spec/workers/move_worker_spec.rb create mode 100644 spec/workers/unfollow_follow_worker_spec.rb (limited to 'app/workers') diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index 1941c2e2d..dc47804c0 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -8,7 +8,7 @@ class FollowService < BaseService # @param [Account] source_account From which to follow # @param [String, Account] uri User URI to follow in the form of username@domain (or account record) # @param [true, false, nil] reblogs Whether or not to show reblogs, defaults to true - def call(source_account, target_account, reblogs: nil) + def call(source_account, target_account, reblogs: nil, bypass_locked: false) reblogs = true if reblogs.nil? target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true) @@ -30,7 +30,7 @@ class FollowService < BaseService ActivityTracker.increment('activity:interactions') - if target_account.locked? || source_account.silenced? || target_account.activitypub? + if (target_account.locked? && !bypass_locked) || source_account.silenced? || target_account.activitypub? request_follow(source_account, target_account, reblogs: reblogs) elsif target_account.local? direct_follow(source_account, target_account, reblogs: reblogs) diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb index 83f7090ee..595730226 100644 --- a/app/workers/move_worker.rb +++ b/app/workers/move_worker.rb @@ -7,7 +7,7 @@ class MoveWorker @source_account = Account.find(source_account_id) @target_account = Account.find(target_account_id) - if @target_account.local? + if @target_account.local? && @source_account.local? rewrite_follows! else queue_follow_unfollows! @@ -21,13 +21,17 @@ class MoveWorker def rewrite_follows! @source_account.passive_relationships .where(account: Account.local) + .where.not(account: @target_account.followers.local) + .where.not(account_id: @target_account.id) .in_batches .update_all(target_account_id: @target_account.id) end def queue_follow_unfollows! + bypass_locked = @target_account.local? + @source_account.followers.local.select(:id).find_in_batches do |accounts| - UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id] } + UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id, bypass_locked] } end end end diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb index 95549e107..b6e665a41 100644 --- a/app/workers/unfollow_follow_worker.rb +++ b/app/workers/unfollow_follow_worker.rb @@ -5,12 +5,15 @@ class UnfollowFollowWorker sidekiq_options queue: 'pull' - def perform(follower_account_id, old_target_account_id, new_target_account_id) + def perform(follower_account_id, old_target_account_id, new_target_account_id, bypass_locked = false) follower_account = Account.find(follower_account_id) old_target_account = Account.find(old_target_account_id) new_target_account = Account.find(new_target_account_id) - FollowService.new.call(follower_account, new_target_account) + follow = follower_account.active_relationships.find_by(target_account: old_target_account) + reblogs = follow&.show_reblogs? + + FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, bypass_locked: bypass_locked) UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true) rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError true diff --git a/spec/workers/move_worker_spec.rb b/spec/workers/move_worker_spec.rb new file mode 100644 index 000000000..b8f4d9900 --- /dev/null +++ b/spec/workers/move_worker_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe MoveWorker do + let(:local_follower) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + let(:source_account) { Fabricate(:account, protocol: :activitypub, domain: 'example.com') } + let(:target_account) { Fabricate(:account, protocol: :activitypub, domain: 'example.com') } + + subject { described_class.new } + + before do + local_follower.follow!(source_account) + end + + context 'both accounts are distant' do + describe 'perform' do + it 'calls UnfollowFollowWorker' do + allow(UnfollowFollowWorker).to receive(:push_bulk) + subject.perform(source_account.id, target_account.id) + expect(UnfollowFollowWorker).to have_received(:push_bulk).with([local_follower.id]) + end + end + end + + context 'target account is local' do + let(:target_account) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account } + + describe 'perform' do + it 'calls UnfollowFollowWorker' do + allow(UnfollowFollowWorker).to receive(:push_bulk) + subject.perform(source_account.id, target_account.id) + expect(UnfollowFollowWorker).to have_received(:push_bulk).with([local_follower.id]) + end + end + end + + context 'both target and source accounts are local' do + let(:target_account) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account } + let(:source_account) { Fabricate(:user, email: 'alice_@example.com', account: Fabricate(:account, username: 'alice_')).account } + + describe 'perform' do + it 'calls makes local followers follow the target account' do + subject.perform(source_account.id, target_account.id) + expect(local_follower.following?(target_account)).to be true + end + + it 'does not fail when a local user is already following both accounts' do + double_follower = Fabricate(:user, email: 'eve@example.com', account: Fabricate(:account, username: 'eve')).account + double_follower.follow!(source_account) + double_follower.follow!(target_account) + subject.perform(source_account.id, target_account.id) + expect(local_follower.following?(target_account)).to be true + end + + it 'does not allow the moved account to follow themselves' do + source_account.follow!(target_account) + subject.perform(source_account.id, target_account.id) + expect(target_account.following?(target_account)).to be false + end + end + end +end diff --git a/spec/workers/unfollow_follow_worker_spec.rb b/spec/workers/unfollow_follow_worker_spec.rb new file mode 100644 index 000000000..5052c5616 --- /dev/null +++ b/spec/workers/unfollow_follow_worker_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe UnfollowFollowWorker do + let(:local_follower) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + let(:source_account) { Fabricate(:account) } + let(:target_account) { Fabricate(:account) } + let(:show_reblogs) { true } + + subject { described_class.new } + + before do + local_follower.follow!(source_account, reblogs: show_reblogs) + end + + context 'when show_reblogs is true' do + let(:show_reblogs) { true } + + describe 'perform' do + it 'unfollows source account and follows target account' do + subject.perform(local_follower.id, source_account.id, target_account.id) + expect(local_follower.following?(source_account)).to be false + expect(local_follower.following?(target_account)).to be true + end + + it 'preserves show_reblogs' do + subject.perform(local_follower.id, source_account.id, target_account.id) + expect(Follow.find_by(account: local_follower, target_account: target_account).show_reblogs?).to be show_reblogs + end + end + end + + context 'when show_reblogs is false' do + let(:show_reblogs) { false } + + describe 'perform' do + it 'unfollows source account and follows target account' do + subject.perform(local_follower.id, source_account.id, target_account.id) + expect(local_follower.following?(source_account)).to be false + expect(local_follower.following?(target_account)).to be true + end + + it 'preserves show_reblogs' do + subject.perform(local_follower.id, source_account.id, target_account.id) + expect(Follow.find_by(account: local_follower, target_account: target_account).show_reblogs?).to be show_reblogs + end + end + end +end -- cgit