From 707ddf7808f90e3ab042d7642d368c2ce8e95e6f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 22 Jun 2019 00:13:10 +0200 Subject: Change domain blocks to automatically support subdomains (#11138) * Change domain blocks to automatically support subdomains If a more authoritative domain is blocked (example.com), then the same block will be applied to a subdomain (foo.example.com) * Match subdomains of existing accounts when blocking/unblocking domains * Improve code style --- app/lib/activitypub/activity/create.rb | 2 +- app/lib/activitypub/activity/flag.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index f55dd35b2..487e8e91e 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -380,7 +380,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def skip_download? return @skip_download if defined?(@skip_download) - @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media? + @skip_download ||= DomainBlock.reject_media?(@account.domain) end def reply_to_local? diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb index f73b93058..1659bc61f 100644 --- a/app/lib/activitypub/activity/flag.rb +++ b/app/lib/activitypub/activity/flag.rb @@ -23,7 +23,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity private def skip_reports? - DomainBlock.find_by(domain: @account.domain)&.reject_reports? + DomainBlock.reject_reports?(@account.domain) end def object_uris -- cgit From aa9b37822bd39dff771844b7b1ac149e37788d34 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 22 Jun 2019 02:50:36 +0200 Subject: Fix audio not being downloaded from remote servers (#11145) --- app/lib/activitypub/activity/create.rb | 2 +- app/models/media_attachment.rb | 8 ++++++++ app/serializers/initial_state_serializer.rb | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 487e8e91e..00f0dd42d 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -370,7 +370,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def unsupported_media_type?(mime_type) - mime_type.present? && !(MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES).include?(mime_type) + mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type) end def supported_blurhash?(blurhash) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 4c55d4b98..a9b6d96c6 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -177,6 +177,14 @@ class MediaAttachment < ApplicationRecord before_save :set_meta class << self + def supported_mime_types + IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES + end + + def supported_file_extensions + IMAGE_FILE_EXTENSIONS + VIDEO_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS + end + private def file_styles(f) diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 3d825410e..8daf36e0a 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -60,7 +60,7 @@ class InitialStateSerializer < ActiveModel::Serializer end def media_attachments - { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::AUDIO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES + MediaAttachment::AUDIO_MIME_TYPES } + { accept_content_types: MediaAttachment.supported_file_extensions + MediaAttachment.supported_mime_types } end private -- cgit From 63c7fe8e4892b22e80c015bf0ecb04496318623b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 8 Jul 2019 12:03:45 +0200 Subject: Refactor controllers for statuses, accounts, and more (#11249) --- app/controllers/about_controller.rb | 16 +- app/controllers/accounts_controller.rb | 15 +- .../activitypub/collections_controller.rb | 16 +- app/controllers/activitypub/inboxes_controller.rb | 7 +- app/controllers/activitypub/outboxes_controller.rb | 6 +- app/controllers/activitypub/replies_controller.rb | 68 +++++++++ app/controllers/api/proofs_controller.rb | 17 +-- app/controllers/application_controller.rb | 4 - .../concerns/account_controller_concern.rb | 34 +---- app/controllers/concerns/account_owned_concern.rb | 33 +++++ .../concerns/status_controller_concern.rb | 87 +++++++++++ app/controllers/custom_css_controller.rb | 1 + app/controllers/emojis_controller.rb | 5 +- app/controllers/follower_accounts_controller.rb | 2 +- app/controllers/following_accounts_controller.rb | 2 +- app/controllers/home_controller.rb | 2 +- app/controllers/intents_controller.rb | 1 + app/controllers/manifests_controller.rb | 1 + app/controllers/media_controller.rb | 1 - app/controllers/public_timelines_controller.rb | 14 +- app/controllers/remote_follow_controller.rb | 12 +- app/controllers/statuses_controller.rb | 164 ++------------------- app/controllers/tags_controller.rb | 18 ++- app/controllers/well_known/host_meta_controller.rb | 2 +- app/controllers/well_known/webfinger_controller.rb | 9 +- app/lib/activitypub/activity/announce.rb | 2 +- app/lib/activitypub/activity/create.rb | 2 +- app/lib/activitypub/activity/delete.rb | 2 +- app/lib/activitypub/tag_manager.rb | 2 +- app/models/status.rb | 9 +- app/serializers/activitypub/activity_serializer.rb | 3 + app/serializers/activitypub/actor_serializer.rb | 2 + .../activitypub/collection_serializer.rb | 2 + app/serializers/activitypub/emoji_serializer.rb | 2 + app/serializers/activitypub/note_serializer.rb | 2 + app/services/process_hashtags_service.rb | 2 +- app/views/statuses/_simple_status.html.haml | 4 +- config/routes.rb | 3 +- .../concerns/account_controller_concern_spec.rb | 2 +- spec/controllers/statuses_controller_spec.rb | 4 +- spec/requests/link_headers_spec.rb | 8 +- 41 files changed, 299 insertions(+), 289 deletions(-) create mode 100644 app/controllers/activitypub/replies_controller.rb create mode 100644 app/controllers/concerns/account_owned_concern.rb create mode 100644 app/controllers/concerns/status_controller_concern.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb index 52a51fd62..761c7f5cd 100644 --- a/app/controllers/about_controller.rb +++ b/app/controllers/about_controller.rb @@ -3,11 +3,11 @@ class AboutController < ApplicationController layout 'public' - before_action :set_instance_presenter, only: [:show, :more, :terms] + before_action :set_body_classes, only: :show + before_action :set_instance_presenter + before_action :set_expires_in - def show - @hide_navbar = true - end + def show; end def more; end @@ -27,4 +27,12 @@ class AboutController < ApplicationController def set_instance_presenter @instance_presenter = InstancePresenter.new end + + def set_body_classes + @hide_navbar = true + end + + def set_expires_in + expires_in 0, public: true + end end diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 065707378..3184a73cb 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -6,13 +6,13 @@ class AccountsController < ApplicationController include AccountControllerConcern before_action :set_cache_headers + before_action :set_body_classes def show respond_to do |format| format.html do - mark_cacheable! unless user_signed_in? + expires_in 0, public: true unless user_signed_in? - @body_classes = 'with-modals' @pinned_statuses = [] @endorsed_accounts = @account.endorsed_accounts.to_a.sample(4) @@ -32,22 +32,25 @@ class AccountsController < ApplicationController end format.rss do - mark_cacheable! + expires_in 0, public: true @statuses = cache_collection(default_statuses.without_reblogs.without_replies.limit(PAGE_SIZE), Status) render xml: RSS::AccountSerializer.render(@account, @statuses) end format.json do - render_cached_json(['activitypub', 'actor', @account], content_type: 'application/activity+json') do - ActiveModelSerializers::SerializableResource.new(@account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter) - end + expires_in 3.minutes, public: true + render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter end end end private + def set_body_classes + @body_classes = 'with-modals' + end + def show_pinned_statuses? [replies_requested?, media_requested?, tag_requested?, params[:max_id].present?, params[:min_id].present?].none? end diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb index 012c3c538..dd2f111b0 100644 --- a/app/controllers/activitypub/collections_controller.rb +++ b/app/controllers/activitypub/collections_controller.rb @@ -2,29 +2,19 @@ class ActivityPub::CollectionsController < Api::BaseController include SignatureVerification + include AccountOwnedConcern - before_action :set_account before_action :set_size before_action :set_statuses before_action :set_cache_headers def show - render_cached_json(['activitypub', 'collection', @account, params[:id]], content_type: 'application/activity+json') do - ActiveModelSerializers::SerializableResource.new( - collection_presenter, - serializer: ActivityPub::CollectionSerializer, - adapter: ActivityPub::Adapter, - skip_activities: true - ) - end + expires_in 3.minutes, public: true + render json: collection_presenter, content_type: 'application/activity+json', serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, skip_activities: true end private - def set_account - @account = Account.find_local!(params[:account_username]) - end - def set_statuses @statuses = scope_for_collection @statuses = cache_collection(@statuses, Status) diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb index e2cd8eaed..9be0676e1 100644 --- a/app/controllers/activitypub/inboxes_controller.rb +++ b/app/controllers/activitypub/inboxes_controller.rb @@ -3,8 +3,7 @@ class ActivityPub::InboxesController < Api::BaseController include SignatureVerification include JsonLdHelper - - before_action :set_account + include AccountOwnedConcern def create if unknown_deleted_account? @@ -27,8 +26,8 @@ class ActivityPub::InboxesController < Api::BaseController false end - def set_account - @account = Account.find_local!(params[:account_username]) if params[:account_username] + def account_required? + params[:account_username].present? end def body diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb index 5147afbf7..4c0b769f0 100644 --- a/app/controllers/activitypub/outboxes_controller.rb +++ b/app/controllers/activitypub/outboxes_controller.rb @@ -4,8 +4,8 @@ class ActivityPub::OutboxesController < Api::BaseController LIMIT = 20 include SignatureVerification + include AccountOwnedConcern - before_action :set_account before_action :set_statuses before_action :set_cache_headers @@ -17,10 +17,6 @@ class ActivityPub::OutboxesController < Api::BaseController private - def set_account - @account = Account.find_local!(params[:account_username]) - end - def outbox_presenter if page_requested? ActivityPub::CollectionPresenter.new( diff --git a/app/controllers/activitypub/replies_controller.rb b/app/controllers/activitypub/replies_controller.rb new file mode 100644 index 000000000..99b7b310f --- /dev/null +++ b/app/controllers/activitypub/replies_controller.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +class ActivityPub::RepliesController < Api::BaseController + include SignatureAuthentication + include Authorization + include AccountOwnedConcern + + DESCENDANTS_LIMIT = 60 + + before_action :set_status + before_action :set_cache_headers + before_action :set_replies + + def index + render json: replies_collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json', skip_activities: true + end + + private + + def set_status + @status = @account.statuses.find(params[:status_id]) + authorize @status, :show? + rescue Mastodon::NotPermittedError + raise ActiveRecord::RecordNotFound + end + + def set_replies + @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses + @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted]) + @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id]) + end + + def replies_collection_presenter + page = ActivityPub::CollectionPresenter.new( + id: account_status_replies_url(@account, @status, page_params), + type: :unordered, + part_of: account_status_replies_url(@account, @status), + next: next_page, + items: @replies.map { |status| status.local ? status : status.id } + ) + + return page if page_requested? + + ActivityPub::CollectionPresenter.new( + id: account_status_replies_url(@account, @status), + type: :unordered, + first: page + ) + end + + def page_requested? + params[:page] == 'true' + end + + def next_page + account_status_replies_url( + @account, + @status, + page: true, + min_id: @replies&.last&.id, + other_accounts: !(@replies&.last&.account_id == @account.id && @replies.size == DESCENDANTS_LIMIT) + ) + end + + def page_params + params_slice(:other_accounts, :min_id).merge(page: true) + end +end diff --git a/app/controllers/api/proofs_controller.rb b/app/controllers/api/proofs_controller.rb index a84ad2014..a98599eee 100644 --- a/app/controllers/api/proofs_controller.rb +++ b/app/controllers/api/proofs_controller.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true class Api::ProofsController < Api::BaseController - before_action :set_account + include AccountOwnedConcern + before_action :set_provider - before_action :check_account_approval - before_action :check_account_suspension def index render json: @account, serializer: @provider.serializer_class @@ -16,15 +15,7 @@ class Api::ProofsController < Api::BaseController @provider = ProofProvider.find(params[:provider]) || raise(ActiveRecord::RecordNotFound) end - def set_account - @account = Account.find_local!(params[:username]) - end - - def check_account_approval - not_found if @account.user_pending? - end - - def check_account_suspension - gone if @account.suspended? + def username_param + params[:username] end end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index bd8000db0..cc8b8e4da 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -154,8 +154,4 @@ class ApplicationController < ActionController::Base def set_cache_headers response.headers['Vary'] = 'Accept' end - - def mark_cacheable! - expires_in 0, public: true - end end diff --git a/app/controllers/concerns/account_controller_concern.rb b/app/controllers/concerns/account_controller_concern.rb index 1c422096c..287a930da 100644 --- a/app/controllers/concerns/account_controller_concern.rb +++ b/app/controllers/concerns/account_controller_concern.rb @@ -3,24 +3,19 @@ module AccountControllerConcern extend ActiveSupport::Concern + include AccountOwnedConcern + FOLLOW_PER_PAGE = 12 included do layout 'public' - before_action :set_account - before_action :check_account_approval - before_action :check_account_suspension before_action :set_instance_presenter before_action :set_link_headers end private - def set_account - @account = Account.find_local!(username_param) - end - def set_instance_presenter @instance_presenter = InstancePresenter.new end @@ -29,27 +24,15 @@ module AccountControllerConcern response.headers['Link'] = LinkHeader.new( [ webfinger_account_link, - atom_account_url_link, actor_url_link, ] ) end - def username_param - params[:account_username] - end - def webfinger_account_link [ webfinger_account_url, - [%w(rel lrdd), %w(type application/xrd+xml)], - ] - end - - def atom_account_url_link - [ - account_url(@account, format: 'atom'), - [%w(rel alternate), %w(type application/atom+xml)], + [%w(rel lrdd), %w(type application/jrd+json)], ] end @@ -63,15 +46,4 @@ module AccountControllerConcern def webfinger_account_url webfinger_url(resource: @account.to_webfinger_s) end - - def check_account_approval - not_found if @account.user_pending? - end - - def check_account_suspension - if @account.suspended? - expires_in(3.minutes, public: true) - gone - end - end end diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb new file mode 100644 index 000000000..99c240fe9 --- /dev/null +++ b/app/controllers/concerns/account_owned_concern.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module AccountOwnedConcern + extend ActiveSupport::Concern + + included do + before_action :set_account, if: :account_required? + before_action :check_account_approval, if: :account_required? + before_action :check_account_suspension, if: :account_required? + end + + private + + def account_required? + true + end + + def set_account + @account = Account.find_local!(username_param) + end + + def username_param + params[:account_username] + end + + def check_account_approval + not_found if @account.local? && @account.user_pending? + end + + def check_account_suspension + expires_in(3.minutes, public: true) && gone if @account.suspended? + end +end diff --git a/app/controllers/concerns/status_controller_concern.rb b/app/controllers/concerns/status_controller_concern.rb new file mode 100644 index 000000000..62a7cf508 --- /dev/null +++ b/app/controllers/concerns/status_controller_concern.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module StatusControllerConcern + extend ActiveSupport::Concern + + ANCESTORS_LIMIT = 40 + DESCENDANTS_LIMIT = 60 + DESCENDANTS_DEPTH_LIMIT = 20 + + def create_descendant_thread(starting_depth, statuses) + depth = starting_depth + statuses.size + + if depth < DESCENDANTS_DEPTH_LIMIT + { + statuses: statuses, + starting_depth: starting_depth, + } + else + next_status = statuses.pop + + { + statuses: statuses, + starting_depth: starting_depth, + next_status: next_status, + } + end + end + + def set_ancestors + @ancestors = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : [] + @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift + end + + def set_descendants + @max_descendant_thread_id = params[:max_descendant_thread_id]&.to_i + @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i + + descendants = cache_collection( + @status.descendants( + DESCENDANTS_LIMIT, + current_account, + @max_descendant_thread_id, + @since_descendant_thread_id, + DESCENDANTS_DEPTH_LIMIT + ), + Status + ) + + @descendant_threads = [] + + if descendants.present? + statuses = [descendants.first] + starting_depth = 0 + + descendants.drop(1).each_with_index do |descendant, index| + if descendants[index].id == descendant.in_reply_to_id + statuses << descendant + else + @descendant_threads << create_descendant_thread(starting_depth, statuses) + + # The thread is broken, assume it's a reply to the root status + starting_depth = 0 + + # ... unless we can find its ancestor in one of the already-processed threads + @descendant_threads.reverse_each do |descendant_thread| + statuses = descendant_thread[:statuses] + + index = statuses.find_index do |thread_status| + thread_status.id == descendant.in_reply_to_id + end + + if index.present? + starting_depth = descendant_thread[:starting_depth] + index + 1 + break + end + end + + statuses = [descendant] + end + end + + @descendant_threads << create_descendant_thread(starting_depth, statuses) + end + + @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT + end +end diff --git a/app/controllers/custom_css_controller.rb b/app/controllers/custom_css_controller.rb index 6e80feaf8..7f4dcfcfe 100644 --- a/app/controllers/custom_css_controller.rb +++ b/app/controllers/custom_css_controller.rb @@ -6,6 +6,7 @@ class CustomCssController < ApplicationController before_action :set_cache_headers def show + expires 3.minutes, public: true render plain: Setting.custom_css || '', content_type: 'text/css' end end diff --git a/app/controllers/emojis_controller.rb b/app/controllers/emojis_controller.rb index 3feb08132..fe4c19cad 100644 --- a/app/controllers/emojis_controller.rb +++ b/app/controllers/emojis_controller.rb @@ -7,9 +7,8 @@ class EmojisController < ApplicationController def show respond_to do |format| format.json do - render_cached_json(['activitypub', 'emoji', @emoji], content_type: 'application/activity+json') do - ActiveModelSerializers::SerializableResource.new(@emoji, serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter) - end + expires_in 3.minutes, public: true + render json: @emoji, content_type: 'application/activity+json', serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter end end end diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb index 415abe10c..8baa64490 100644 --- a/app/controllers/follower_accounts_controller.rb +++ b/app/controllers/follower_accounts_controller.rb @@ -8,7 +8,7 @@ class FollowerAccountsController < ApplicationController def index respond_to do |format| format.html do - mark_cacheable! unless user_signed_in? + expires_in 0, public: true unless user_signed_in? next if @account.user_hides_network? diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb index 948725664..4d1ea4594 100644 --- a/app/controllers/following_accounts_controller.rb +++ b/app/controllers/following_accounts_controller.rb @@ -8,7 +8,7 @@ class FollowingAccountsController < ApplicationController def index respond_to do |format| format.html do - mark_cacheable! unless user_signed_in? + expires_in 0, public: true unless user_signed_in? next if @account.user_hides_network? diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb index 85622a7b5..d1c525134 100644 --- a/app/controllers/home_controller.rb +++ b/app/controllers/home_controller.rb @@ -21,7 +21,7 @@ class HomeController < ApplicationController when 'statuses' status = Status.find_by(id: matches[2]) - if status && (status.public_visibility? || status.unlisted_visibility?) + if status&.distributable? redirect_to(ActivityPub::TagManager.instance.url_for(status)) return end diff --git a/app/controllers/intents_controller.rb b/app/controllers/intents_controller.rb index 9f41cf48a..ca89fc7fe 100644 --- a/app/controllers/intents_controller.rb +++ b/app/controllers/intents_controller.rb @@ -2,6 +2,7 @@ class IntentsController < ApplicationController before_action :check_uri + rescue_from Addressable::URI::InvalidURIError, with: :handle_invalid_uri def show diff --git a/app/controllers/manifests_controller.rb b/app/controllers/manifests_controller.rb index 332d845d8..1e5db4393 100644 --- a/app/controllers/manifests_controller.rb +++ b/app/controllers/manifests_controller.rb @@ -4,6 +4,7 @@ class ManifestsController < ApplicationController skip_before_action :store_current_location def show + expires_in 3.minutes, public: true render json: InstancePresenter.new, serializer: ManifestSerializer end end diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb index d44b52d26..b3b7519a1 100644 --- a/app/controllers/media_controller.rb +++ b/app/controllers/media_controller.rb @@ -31,7 +31,6 @@ class MediaController < ApplicationController def verify_permitted_status! authorize @media_attachment.status, :show? rescue Mastodon::NotPermittedError - # Reraise in order to get a 404 instead of a 403 error code raise ActiveRecord::RecordNotFound end diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb index 53d4472d8..23506b990 100644 --- a/app/controllers/public_timelines_controller.rb +++ b/app/controllers/public_timelines_controller.rb @@ -8,20 +8,16 @@ class PublicTimelinesController < ApplicationController before_action :set_instance_presenter def show - respond_to do |format| - format.html do - @initial_state_json = ActiveModelSerializers::SerializableResource.new( - InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token), - serializer: InitialStateSerializer - ).to_json - end - end + @initial_state_json = ActiveModelSerializers::SerializableResource.new( + InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token), + serializer: InitialStateSerializer + ).to_json end private def check_enabled - raise ActiveRecord::RecordNotFound unless Setting.timeline_preview + not_found unless Setting.timeline_preview end def set_body_classes diff --git a/app/controllers/remote_follow_controller.rb b/app/controllers/remote_follow_controller.rb index 8ba331cd1..0fb71d335 100644 --- a/app/controllers/remote_follow_controller.rb +++ b/app/controllers/remote_follow_controller.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true class RemoteFollowController < ApplicationController + include AccountOwnedConcern + layout 'modal' - before_action :set_account - before_action :gone, if: :suspended_account? before_action :set_body_classes def new @@ -32,14 +32,6 @@ class RemoteFollowController < ApplicationController { acct: session[:remote_follow] } end - def set_account - @account = Account.find_local!(params[:account_username]) - end - - def suspended_account? - @account.suspended? - end - def set_body_classes @body_classes = 'modal-layout' @hide_header = true diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index 776099ca8..13ce5c691 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -1,24 +1,21 @@ # frozen_string_literal: true class StatusesController < ApplicationController + include StatusControllerConcern include SignatureAuthentication include Authorization - - ANCESTORS_LIMIT = 40 - DESCENDANTS_LIMIT = 60 - DESCENDANTS_DEPTH_LIMIT = 20 + include AccountOwnedConcern layout 'public' - before_action :set_account before_action :set_status before_action :set_instance_presenter before_action :set_link_headers - before_action :check_account_suspension before_action :redirect_to_original, only: [:show] before_action :set_referrer_policy_header, only: [:show] before_action :set_cache_headers - before_action :set_replies, only: [:replies] + before_action :set_body_classes + before_action :set_autoplay, only: :embed content_security_policy only: :embed do |p| p.frame_ancestors(false) @@ -28,25 +25,20 @@ class StatusesController < ApplicationController respond_to do |format| format.html do expires_in 10.seconds, public: true if current_account.nil? - - @body_classes = 'with-modals' - set_ancestors set_descendants end format.json do - render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: @status.distributable?) do - ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter) - end + expires_in 3.minutes, public: @status.distributable? + render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter end end end def activity - render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: @status.distributable?) do - ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter) - end + expires_in 3.minutes, public: @status.distributable? + render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter end def embed @@ -54,120 +46,14 @@ class StatusesController < ApplicationController expires_in 180, public: true response.headers['X-Frame-Options'] = 'ALLOWALL' - @autoplay = ActiveModel::Type::Boolean.new.cast(params[:autoplay]) render layout: 'embedded' end - def replies - render json: replies_collection_presenter, - serializer: ActivityPub::CollectionSerializer, - adapter: ActivityPub::Adapter, - content_type: 'application/activity+json', - skip_activities: true - end - private - def replies_collection_presenter - page = ActivityPub::CollectionPresenter.new( - id: replies_account_status_url(@account, @status, page_params), - type: :unordered, - part_of: replies_account_status_url(@account, @status), - next: next_page, - items: @replies.map { |status| status.local ? status : status.id } - ) - if page_requested? - page - else - ActivityPub::CollectionPresenter.new( - id: replies_account_status_url(@account, @status), - type: :unordered, - first: page - ) - end - end - - def create_descendant_thread(starting_depth, statuses) - depth = starting_depth + statuses.size - - if depth < DESCENDANTS_DEPTH_LIMIT - { - statuses: statuses, - starting_depth: starting_depth, - } - else - next_status = statuses.pop - - { - statuses: statuses, - starting_depth: starting_depth, - next_status: next_status, - } - end - end - - def set_account - @account = Account.find_local!(params[:account_username]) - end - - def set_ancestors - @ancestors = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : [] - @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift - end - - def set_descendants - @max_descendant_thread_id = params[:max_descendant_thread_id]&.to_i - @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i - - descendants = cache_collection( - @status.descendants( - DESCENDANTS_LIMIT, - current_account, - @max_descendant_thread_id, - @since_descendant_thread_id, - DESCENDANTS_DEPTH_LIMIT - ), - Status - ) - - @descendant_threads = [] - - if descendants.present? - statuses = [descendants.first] - starting_depth = 0 - - descendants.drop(1).each_with_index do |descendant, index| - if descendants[index].id == descendant.in_reply_to_id - statuses << descendant - else - @descendant_threads << create_descendant_thread(starting_depth, statuses) - - # The thread is broken, assume it's a reply to the root status - starting_depth = 0 - - # ... unless we can find its ancestor in one of the already-processed threads - @descendant_threads.reverse_each do |descendant_thread| - statuses = descendant_thread[:statuses] - - index = statuses.find_index do |thread_status| - thread_status.id == descendant.in_reply_to_id - end - - if index.present? - starting_depth = descendant_thread[:starting_depth] + index + 1 - break - end - end - - statuses = [descendant] - end - end - - @descendant_threads << create_descendant_thread(starting_depth, statuses) - end - - @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT + def set_body_classes + @body_classes = 'with-modals' end def set_link_headers @@ -185,39 +71,15 @@ class StatusesController < ApplicationController @instance_presenter = InstancePresenter.new end - def check_account_suspension - gone if @account.suspended? - end - def redirect_to_original redirect_to ActivityPub::TagManager.instance.url_for(@status.reblog) if @status.reblog? end def set_referrer_policy_header - return if @status.public_visibility? || @status.unlisted_visibility? - response.headers['Referrer-Policy'] = 'origin' - end - - def page_requested? - params[:page] == 'true' - end - - def set_replies - @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses - @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted]) - @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id]) - end - - def next_page - last_reply = @replies.last - return if last_reply.nil? - same_account = last_reply.account_id == @account.id - return unless same_account || @replies.size == DESCENDANTS_LIMIT - same_account = false unless @replies.size == DESCENDANTS_LIMIT - replies_account_status_url(@account, @status, page: true, min_id: last_reply.id, other_accounts: !same_account) + response.headers['Referrer-Policy'] = 'origin' unless @status.distributable? end - def page_params - { page: true, other_accounts: params[:other_accounts], min_id: params[:min_id] }.compact + def set_autoplay + @autoplay = truthy_param?(:autoplay) end end diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb index 66b184901..2ecce0ca2 100644 --- a/app/controllers/tags_controller.rb +++ b/app/controllers/tags_controller.rb @@ -5,14 +5,15 @@ class TagsController < ApplicationController layout 'public' + before_action :set_tag before_action :set_body_classes before_action :set_instance_presenter def show - @tag = Tag.find_normalized!(params[:id]) - respond_to do |format| format.html do + expires_in 0, public: true + @initial_state_json = ActiveModelSerializers::SerializableResource.new( InitialStatePresenter.new(settings: {}, token: current_session&.token), serializer: InitialStateSerializer @@ -20,6 +21,8 @@ class TagsController < ApplicationController end format.rss do + expires_in 0, public: true + @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none)).limit(PAGE_SIZE) @statuses = cache_collection(@statuses, Status) @@ -27,19 +30,22 @@ class TagsController < ApplicationController end format.json do + expires_in 3.minutes, public: true + @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none), current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id]) @statuses = cache_collection(@statuses, Status) - render json: collection_presenter, - serializer: ActivityPub::CollectionSerializer, - adapter: ActivityPub::Adapter, - content_type: 'application/activity+json' + render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json' end end end private + def set_tag + @tag = Tag.find_normalized!(params[:id]) + end + def set_body_classes @body_classes = 'with-modals' end diff --git a/app/controllers/well_known/host_meta_controller.rb b/app/controllers/well_known/host_meta_controller.rb index 5fb70288a..2e9298c4a 100644 --- a/app/controllers/well_known/host_meta_controller.rb +++ b/app/controllers/well_known/host_meta_controller.rb @@ -13,7 +13,7 @@ module WellKnown format.xml { render content_type: 'application/xrd+xml' } end - expires_in(3.days, public: true) + expires_in 3.days, public: true end end end diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb index 28654b61d..53f7f1e27 100644 --- a/app/controllers/well_known/webfinger_controller.rb +++ b/app/controllers/well_known/webfinger_controller.rb @@ -19,7 +19,7 @@ module WellKnown end end - expires_in(3.days, public: true) + expires_in 3.days, public: true rescue ActiveRecord::RecordNotFound head 404 end @@ -27,12 +27,9 @@ module WellKnown private def username_from_resource - resource_user = resource_param - + resource_user = resource_param username, domain = resource_user.split('@') - if Rails.configuration.x.alternate_domains.include?(domain) - resource_user = "#{username}@#{Rails.configuration.x.local_domain}" - end + resource_user = "#{username}@#{Rails.configuration.x.local_domain}" if Rails.configuration.x.alternate_domains.include?(domain) WebfingerResource.new(resource_user).username end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 1aa6ee9ec..34c646668 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -40,7 +40,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity end def announceable?(status) - status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility? + status.account_id == @account.id || status.distributable? end def related_to_local_activity? diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 00f0dd42d..5849c20d7 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -42,7 +42,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) distribute(@status) - forward_for_reply if @status.public_visibility? || @status.unlisted_visibility? + forward_for_reply if @status.distributable? end def find_existing_status diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 0eb14b89c..1f2b40c15 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -31,7 +31,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity return if @status.nil? - if @status.public_visibility? || @status.unlisted_visibility? + if @status.distributable? forward_for_reply forward_for_reblogs end diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 595291342..4d452f290 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -51,7 +51,7 @@ class ActivityPub::TagManager def replies_uri_for(target, page_params = nil) raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local? - replies_account_status_url(target.account, target, page_params) + account_status_replies_url(target.account, target, page_params) end # Primary audience of a status diff --git a/app/models/status.rb b/app/models/status.rb index 906756e85..6f1e35e4a 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -193,7 +193,7 @@ class Status < ApplicationRecord end def hidden? - private_visibility? || direct_visibility? || limited_visibility? + !distributable? end def distributable? @@ -446,7 +446,8 @@ class Status < ApplicationRecord end def update_statistics - return unless public_visibility? || unlisted_visibility? + return unless distributable? + ActivityTracker.increment('activity:statuses:local') end @@ -455,7 +456,7 @@ class Status < ApplicationRecord account&.increment_count!(:statuses_count) reblog&.increment_count!(:reblogs_count) if reblog? - thread&.increment_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?) + thread&.increment_count!(:replies_count) if in_reply_to_id.present? && distributable? end def decrement_counter_caches @@ -463,7 +464,7 @@ class Status < ApplicationRecord account&.decrement_count!(:statuses_count) reblog&.decrement_count!(:reblogs_count) if reblog? - thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?) + thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && distributable? end def unlink_from_conversations diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb index c06d5c87c..fdedbc9d1 100644 --- a/app/serializers/activitypub/activity_serializer.rb +++ b/app/serializers/activitypub/activity_serializer.rb @@ -1,9 +1,12 @@ # frozen_string_literal: true class ActivityPub::ActivitySerializer < ActivityPub::Serializer + cache key: 'activity', expires_in: 3.minutes + attributes :id, :type, :actor, :published, :to, :cc has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, if: :serialize_object? + attribute :proper_uri, key: :object, unless: :serialize_object? attribute :atom_uri, if: :announce? diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index 0644219fb..ab7be27f6 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -3,6 +3,8 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer include RoutingHelper + cache key: 'actor', expires_in: 3.minutes + context :security context_extensions :manually_approves_followers, :featured, :also_known_as, diff --git a/app/serializers/activitypub/collection_serializer.rb b/app/serializers/activitypub/collection_serializer.rb index da1ba735f..9dd8134d3 100644 --- a/app/serializers/activitypub/collection_serializer.rb +++ b/app/serializers/activitypub/collection_serializer.rb @@ -7,6 +7,8 @@ class ActivityPub::CollectionSerializer < ActivityPub::Serializer super end + cache key: 'collection', expires_in: 3.minutes + attribute :id, if: -> { object.id.present? } attribute :type attribute :total_items, if: -> { object.size.present? } diff --git a/app/serializers/activitypub/emoji_serializer.rb b/app/serializers/activitypub/emoji_serializer.rb index 4dc38f3ea..08df25d7d 100644 --- a/app/serializers/activitypub/emoji_serializer.rb +++ b/app/serializers/activitypub/emoji_serializer.rb @@ -3,6 +3,8 @@ class ActivityPub::EmojiSerializer < ActivityPub::Serializer include RoutingHelper + cache key: 'emoji', expires_in: 3.minutes + context_extensions :emoji attributes :id, :type, :name, :updated diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 67f596e78..87acc5429 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::NoteSerializer < ActivityPub::Serializer + cache key: 'note', expires_in: 3.minutes + context_extensions :atom_uri, :conversation, :sensitive, :hashtag, :emoji, :focal_point, :blurhash diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index d5ec076a8..b6974e598 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -14,7 +14,7 @@ class ProcessHashtagsService < BaseService TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility? end - return unless status.public_visibility? || status.unlisted_visibility? + return unless status.distributable? status.account.featured_tags.where(tag_id: records.map(&:id)).each do |featured_tag| featured_tag.increment(status.created_at) diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml index 11220dfcb..38fde1be8 100644 --- a/app/views/statuses/_simple_status.html.haml +++ b/app/views/statuses/_simple_status.html.haml @@ -50,9 +50,9 @@ = fa_icon 'reply-all fw' .status__action-bar__counter__label= obscured_counter status.replies_count = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do - - if status.public_visibility? || status.unlisted_visibility? + - if status.distributable? = fa_icon 'retweet fw' - - elsif status.private_visibility? + - elsif status.private_visibility? || status.limited_visibility? = fa_icon 'lock fw' - else = fa_icon 'envelope fw' diff --git a/config/routes.rb b/config/routes.rb index 69b495a96..115e7bb44 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -52,8 +52,9 @@ Rails.application.routes.draw do member do get :activity get :embed - get :replies end + + resources :replies, only: [:index], module: :activitypub end resources :followers, only: [:index], controller: :follower_accounts diff --git a/spec/controllers/concerns/account_controller_concern_spec.rb b/spec/controllers/concerns/account_controller_concern_spec.rb index ea2b4a2a1..7ea214a7d 100644 --- a/spec/controllers/concerns/account_controller_concern_spec.rb +++ b/spec/controllers/concerns/account_controller_concern_spec.rb @@ -41,7 +41,7 @@ describe ApplicationController, type: :controller do it 'sets link headers' do account = Fabricate(:account, username: 'username', user: Fabricate(:user)) get 'success', params: { account_username: 'username' } - expect(response.headers['Link'].to_s).to eq '; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json"' + expect(response.headers['Link'].to_s).to eq '; rel="lrdd"; type="application/jrd+json", ; rel="alternate"; type="application/activity+json"' end it 'returns http success' do diff --git a/spec/controllers/statuses_controller_spec.rb b/spec/controllers/statuses_controller_spec.rb index 95e5c363c..6905dae10 100644 --- a/spec/controllers/statuses_controller_spec.rb +++ b/spec/controllers/statuses_controller_spec.rb @@ -92,7 +92,7 @@ describe StatusesController do end it 'assigns @max_descendant_thread_id for the last thread if it is hitting the status limit' do - stub_const 'StatusesController::DESCENDANTS_LIMIT', 1 + stub_const 'StatusControllerConcern::DESCENDANTS_LIMIT', 1 status = Fabricate(:status) child = Fabricate(:status, in_reply_to_id: status.id) @@ -103,7 +103,7 @@ describe StatusesController do end it 'assigns @descendant_threads for threads with :next_status key if they are hitting the depth limit' do - stub_const 'StatusesController::DESCENDANTS_DEPTH_LIMIT', 2 + stub_const 'StatusControllerConcern::DESCENDANTS_DEPTH_LIMIT', 2 status = Fabricate(:status) child0 = Fabricate(:status, in_reply_to_id: status.id) child1 = Fabricate(:status, in_reply_to_id: child0.id) diff --git a/spec/requests/link_headers_spec.rb b/spec/requests/link_headers_spec.rb index 3dc408d92..712ee262b 100644 --- a/spec/requests/link_headers_spec.rb +++ b/spec/requests/link_headers_spec.rb @@ -11,16 +11,16 @@ describe 'Link headers' do end it 'contains webfinger url in link header' do - link_header = link_header_with_type('application/xrd+xml') + link_header = link_header_with_type('application/jrd+json') expect(link_header.href).to match 'http://www.example.com/.well-known/webfinger?resource=acct%3Atest%40cb6e6126.ngrok.io' expect(link_header.attr_pairs.first).to eq %w(rel lrdd) end - it 'contains atom url in link header' do - link_header = link_header_with_type('application/atom+xml') + it 'contains activitypub url in link header' do + link_header = link_header_with_type('application/activity+json') - expect(link_header.href).to eq 'http://www.example.com/users/test.atom' + expect(link_header.href).to eq 'https://cb6e6126.ngrok.io/users/test' expect(link_header.attr_pairs.first).to eq %w(rel alternate) end -- cgit From 6ff67be0f6e79ec403e08c69717ee8c89451c70e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 13 Jul 2019 16:45:50 +0200 Subject: Add a spam check (#11217) * Add a spam check * Use Nilsimsa to generate locality-sensitive hashes and compare using Levenshtein distance * Add more tests * Add exemption when the message is a reply to something that mentions the sender * Use Nilsimsa Compare Value instead of Levenshtein distance * Use MD5 for messages shorter than 10 characters * Add message to automated report, do not add non-public statuses to automated report, add trust level to accounts and make unsilencing raise the trust level to prevent repeated spam checks on that account * Expire spam check data after 3 months * Add support for local statuses, reduce expiration to 1 week, always create a report * Add content warnings to the spam check and exempt empty statuses * Change Nilsimsa threshold to 95 and make sure removed statuses are removed from the spam check * Add all matched statuses into automatic report --- Gemfile | 1 + Gemfile.lock | 8 + app/lib/activitypub/activity/create.rb | 13 ++ app/lib/spam_check.rb | 169 +++++++++++++++++++++ app/models/account.rb | 18 ++- app/services/remove_status_service.rb | 5 + config/locales/en.yml | 2 + .../20190701022101_add_trust_level_to_accounts.rb | 5 + db/schema.rb | 1 + spec/lib/spam_check_spec.rb | 160 +++++++++++++++++++ 10 files changed, 377 insertions(+), 5 deletions(-) create mode 100644 app/lib/spam_check.rb create mode 100644 db/migrate/20190701022101_add_trust_level_to_accounts.rb create mode 100644 spec/lib/spam_check_spec.rb (limited to 'app/lib/activitypub/activity') diff --git a/Gemfile b/Gemfile index 613515628..15334678b 100644 --- a/Gemfile +++ b/Gemfile @@ -58,6 +58,7 @@ gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.1' gem 'link_header', '~> 0.0' gem 'mime-types', '~> 3.2', require: 'mime/types/columnar' +gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532' gem 'nokogiri', '~> 1.10' gem 'nsa', '~> 0.2' gem 'oj', '~> 3.7' diff --git a/Gemfile.lock b/Gemfile.lock index 340bbcdd8..c3198b7d9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,6 +12,13 @@ GIT specs: http_parser.rb (0.6.1) +GIT + remote: https://github.com/witgo/nilsimsa + revision: fd184883048b922b176939f851338d0a4971a532 + ref: fd184883048b922b176939f851338d0a4971a532 + specs: + nilsimsa (1.1.2) + GEM remote: https://rubygems.org/ specs: @@ -704,6 +711,7 @@ DEPENDENCIES microformats (~> 4.1) mime-types (~> 3.2) net-ldap (~> 0.10) + nilsimsa! nokogiri (~> 1.10) nsa (~> 0.2) oj (~> 3.7) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 5849c20d7..56c24680a 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -41,6 +41,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) + check_for_spam distribute(@status) forward_for_reply if @status.distributable? end @@ -406,6 +407,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Account.local.where(username: local_usernames).exists? end + def check_for_spam + spam_check = SpamCheck.new(@status) + + return if spam_check.skip? + + if spam_check.spam? + spam_check.flag! + else + spam_check.remember! + end + end + def forward_for_reply return unless @json['signature'].present? && reply_to_local? ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb new file mode 100644 index 000000000..923d48a02 --- /dev/null +++ b/app/lib/spam_check.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +class SpamCheck + include Redisable + include ActionView::Helpers::TextHelper + + NILSIMSA_COMPARE_THRESHOLD = 95 + NILSIMSA_MIN_SIZE = 10 + EXPIRE_SET_AFTER = 1.week.seconds + + def initialize(status) + @account = status.account + @status = status + end + + def skip? + already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply? + end + + def spam? + if insufficient_data? + false + elsif nilsimsa? + any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } + else + any_other_digest?('md5') { |_, other_digest| other_digest == digest } + end + end + + def flag! + auto_silence_account! + auto_report_status! + end + + def remember! + # The scores in sorted sets don't actually have enough bits to hold an exact + # value of our snowflake IDs, so we use it only for its ordering property. To + # get the correct status ID back, we have to save it in the string value + + redis.zadd(redis_key, @status.id, digest_with_algorithm) + redis.zremrangebyrank(redis_key, '0', '-10') + redis.expire(redis_key, EXPIRE_SET_AFTER) + end + + def reset! + redis.del(redis_key) + end + + def hashable_text + return @hashable_text if defined?(@hashable_text) + + @hashable_text = @status.text + @hashable_text = remove_mentions(@hashable_text) + @hashable_text = strip_tags(@hashable_text) unless @status.local? + @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text) + @hashable_text = remove_whitespace(@hashable_text) + end + + def insufficient_data? + hashable_text.blank? + end + + def digest + @digest ||= begin + if nilsimsa? + Nilsimsa.new(hashable_text).hexdigest + else + Digest::MD5.hexdigest(hashable_text) + end + end + end + + def digest_with_algorithm + if nilsimsa? + ['nilsimsa', digest, @status.id].join(':') + else + ['md5', digest, @status.id].join(':') + end + end + + private + + def remove_mentions(text) + return text.gsub(Account::MENTION_RE, '') if @status.local? + + Nokogiri::HTML.fragment(text).tap do |html| + mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) } + + html.traverse do |element| + element.unlink if element.name == 'a' && mentions.include?(element['href']) + end + end.to_s + end + + def normalize_unicode(text) + text.unicode_normalize(:nfkc).downcase + end + + def remove_whitespace(text) + text.gsub(/\s+/, ' ').strip + end + + def auto_silence_account! + @account.silence! + end + + def auto_report_status! + status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable? + ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced')) + end + + def already_flagged? + @account.silenced? + end + + def trusted? + @account.trust_level > Account::TRUST_LEVELS[:untrusted] + end + + def no_unsolicited_mentions? + @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) } + end + + def solicited_reply? + !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists? + end + + def nilsimsa_compare_value(first, second) + first = [first].pack('H*') + second = [second].pack('H*') + bits = 0 + + 0.upto(31) do |i| + bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord + end + + 128 - bits # -128 <= Nilsimsa Compare Value <= 128 + end + + def nilsimsa? + hashable_text.size > NILSIMSA_MIN_SIZE + end + + def other_digests + redis.zrange(redis_key, 0, -1) + end + + def any_other_digest?(filter_algorithm) + other_digests.any? do |record| + algorithm, other_digest, status_id = record.split(':') + + next unless algorithm == filter_algorithm + + yield algorithm, other_digest, status_id + end + end + + def matching_status_ids + if nilsimsa? + other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact + else + other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact + end + end + + def redis_key + @redis_key ||= "spam_check:#{@account.id}" + end +end diff --git a/app/models/account.rb b/app/models/account.rb index d6772eb98..a22b7fd7c 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -45,6 +45,7 @@ # also_known_as :string is an Array # silenced_at :datetime # suspended_at :datetime +# trust_level :integer # class Account < ApplicationRecord @@ -62,6 +63,11 @@ class Account < ApplicationRecord include AccountCounters include DomainNormalizable + TRUST_LEVELS = { + untrusted: 0, + trusted: 1, + }.freeze + enum protocol: [:ostatus, :activitypub] validates :username, presence: true @@ -163,6 +169,10 @@ class Account < ApplicationRecord last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago end + def trust_level + self[:trust_level] || 0 + end + def refresh! ResolveAccountService.new.call(acct) unless local? end @@ -171,21 +181,19 @@ class Account < ApplicationRecord silenced_at.present? end - def silence!(date = nil) - date ||= Time.now.utc + def silence!(date = Time.now.utc) update!(silenced_at: date) end def unsilence! - update!(silenced_at: nil) + update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level) end def suspended? suspended_at.present? end - def suspend!(date = nil) - date ||= Time.now.utc + def suspend!(date = Time.now.utc) transaction do user&.disable! if local? update!(suspended_at: date) diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 6311971ff..a69fce8b8 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -23,6 +23,7 @@ class RemoveStatusService < BaseService remove_from_hashtags remove_from_public remove_from_media if status.media_attachments.any? + remove_from_spam_check @status.destroy! else @@ -142,6 +143,10 @@ class RemoveStatusService < BaseService redis.publish('timeline:public:local:media', @payload) if @status.local? end + def remove_from_spam_check + redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id) + end + def lock_options { redis: Redis.current, key: "distribute:#{@status.id}" } end diff --git a/config/locales/en.yml b/config/locales/en.yml index 00b7d1dbe..89251ad40 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -875,6 +875,8 @@ en: profile: Profile relationships: Follows and followers two_factor_authentication: Two-factor Auth + spam_check: + spam_detected_and_silenced: This is an automated report. Spam has been detected and the sender has been silenced automatically. If this is a mistake, please unsilence the account. statuses: attached: description: 'Attached: %{attached}' diff --git a/db/migrate/20190701022101_add_trust_level_to_accounts.rb b/db/migrate/20190701022101_add_trust_level_to_accounts.rb new file mode 100644 index 000000000..917486d2e --- /dev/null +++ b/db/migrate/20190701022101_add_trust_level_to_accounts.rb @@ -0,0 +1,5 @@ +class AddTrustLevelToAccounts < ActiveRecord::Migration[5.2] + def change + add_column :accounts, :trust_level, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 2e38fb1f2..c7b6b9be6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -148,6 +148,7 @@ ActiveRecord::Schema.define(version: 2019_07_06_233204) do t.string "also_known_as", array: true t.datetime "silenced_at" t.datetime "suspended_at" + t.integer "trust_level" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id" diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb new file mode 100644 index 000000000..c722dc642 --- /dev/null +++ b/spec/lib/spam_check_spec.rb @@ -0,0 +1,160 @@ +require 'rails_helper' + +RSpec.describe SpamCheck do + let!(:sender) { Fabricate(:account) } + let!(:alice) { Fabricate(:account, username: 'alice') } + let!(:bob) { Fabricate(:account, username: 'bob') } + + def status_with_html(text, options = {}) + status = PostStatusService.new.call(sender, { text: text }.merge(options)) + status.update_columns(text: Formatter.instance.format(status), local: false) + status + end + + describe '#hashable_text' do + it 'removes mentions from HTML for remote statuses' do + status = status_with_html('@alice Hello') + expect(described_class.new(status).hashable_text).to eq 'hello' + end + + it 'removes mentions from text for local statuses' do + status = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?") + expect(described_class.new(status).hashable_text).to eq 'hey , how are you?' + end + end + + describe '#insufficient_data?' do + it 'returns true when there is no text' do + status = status_with_html('@alice') + expect(described_class.new(status).insufficient_data?).to be true + end + + it 'returns false when there is text' do + status = status_with_html('@alice h') + expect(described_class.new(status).insufficient_data?).to be false + end + end + + describe '#digest' do + it 'returns a string' do + status = status_with_html('@alice Hello world') + expect(described_class.new(status).digest).to be_a String + end + end + + describe '#spam?' do + it 'returns false for a unique status' do + status = status_with_html('@alice Hello') + expect(described_class.new(status).spam?).to be false + end + + it 'returns false for different statuses to the same recipient' do + status1 = status_with_html('@alice Hello') + described_class.new(status1).remember! + status2 = status_with_html('@alice Are you available to talk?') + expect(described_class.new(status2).spam?).to be false + end + + it 'returns false for statuses with different content warnings' do + status1 = status_with_html('@alice Are you available to talk?') + described_class.new(status1).remember! + status2 = status_with_html('@alice Are you available to talk?', spoiler_text: 'This is a completely different matter than what I was talking about previously, I swear!') + expect(described_class.new(status2).spam?).to be false + end + + it 'returns false for different statuses to different recipients' do + status1 = status_with_html('@alice How is it going?') + described_class.new(status1).remember! + status2 = status_with_html('@bob Are you okay?') + expect(described_class.new(status2).spam?).to be false + end + + it 'returns false for very short different statuses to different recipients' do + status1 = status_with_html('@alice 🙄') + described_class.new(status1).remember! + status2 = status_with_html('@bob Huh?') + expect(described_class.new(status2).spam?).to be false + end + + it 'returns false for statuses with no text' do + status1 = status_with_html('@alice') + described_class.new(status1).remember! + status2 = status_with_html('@bob') + expect(described_class.new(status2).spam?).to be false + end + + it 'returns true for duplicate statuses to the same recipient' do + status1 = status_with_html('@alice Hello') + described_class.new(status1).remember! + status2 = status_with_html('@alice Hello') + expect(described_class.new(status2).spam?).to be true + end + + it 'returns true for duplicate statuses to different recipients' do + status1 = status_with_html('@alice Hello') + described_class.new(status1).remember! + status2 = status_with_html('@bob Hello') + expect(described_class.new(status2).spam?).to be true + end + + it 'returns true for nearly identical statuses with random numbers' do + source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.' + status1 = status_with_html('@alice ' + source_text + ' 1234') + described_class.new(status1).remember! + status2 = status_with_html('@bob ' + source_text + ' 9568') + expect(described_class.new(status2).spam?).to be true + end + end + + describe '#skip?' do + it 'returns true when the sender is already silenced' do + status = status_with_html('@alice Hello') + sender.silence! + expect(described_class.new(status).skip?).to be true + end + + it 'returns true when the mentioned person follows the sender' do + status = status_with_html('@alice Hello') + alice.follow!(sender) + expect(described_class.new(status).skip?).to be true + end + + it 'returns false when even one mentioned person doesn\'t follow the sender' do + status = status_with_html('@alice @bob Hello') + alice.follow!(sender) + expect(described_class.new(status).skip?).to be false + end + + it 'returns true when the sender is replying to a status that mentions the sender' do + parent = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?") + status = status_with_html('@alice @bob Hello', thread: parent) + expect(described_class.new(status).skip?).to be true + end + end + + describe '#remember!' do + pending + end + + describe '#flag!' do + let!(:status1) { status_with_html('@alice General Kenobi you are a bold one') } + let!(:status2) { status_with_html('@alice @bob General Kenobi, you are a bold one') } + + before do + described_class.new(status1).remember! + described_class.new(status2).flag! + end + + it 'silences the account' do + expect(sender.silenced?).to be true + end + + it 'creates a report about the account' do + expect(sender.targeted_reports.unresolved.count).to eq 1 + end + + it 'attaches both matching statuses to the report' do + expect(sender.targeted_reports.first.status_ids).to include(status1.id, status2.id) + end + end +end -- cgit From 730c4053d642024b9949d72c8a9f1873532c6212 Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 19 Jul 2019 01:44:42 +0200 Subject: Add ActivityPub actor representing the entire server (#11321) * Add support for an instance actor * Skip username validation for local Application accounts * Add migration script to create instance actor * Make Codeclimate happy * Switch to id -99 for instance actor * Remove unused `icon` and `image` attributes from instance actor * Use if/elsif/else instead of return + ternary operator * Add instance actor to fresh installs * Use instance actor as instance representative Use instance actor for forwarding reports, relay operations, and spam auto-reporting. * Seed database in test environment * Fix single-user mode * Fix tests * Fix specs to accomodate for an extra `Account` * Auto-reject follows on instance actor Following an instance actor might make sense, but we are not handling that right now, so auto-reject. * Fix webfinger lookup and serialization for instance actor * Rename instance actor * Make it clear in the HTML view that the instance actor should not be blocked * Raise cache time for instance actor as there's no dynamic content * Re-use /about/more with a flash message for instance actor profile --- app/controllers/about_controller.rb | 4 +- app/controllers/application_controller.rb | 2 +- app/controllers/home_controller.rb | 2 +- app/controllers/instance_actors_controller.rb | 20 ++++++++ app/javascript/styles/mastodon/containers.scss | 4 ++ app/lib/activitypub/activity/follow.rb | 2 +- app/lib/activitypub/tag_manager.rb | 5 +- app/lib/webfinger_resource.rb | 6 +++ app/models/account.rb | 8 ++- app/models/concerns/account_finder_concern.rb | 2 +- app/serializers/activitypub/actor_serializer.rb | 14 ++++-- app/serializers/webfinger_serializer.rb | 25 +++++++--- app/views/about/more.html.haml | 2 + app/views/well_known/webfinger/show.xml.ruby | 57 ++++++++++++++-------- config/locales/en.yml | 3 ++ config/routes.rb | 4 ++ db/migrate/20190715164535_add_instance_actor.rb | 9 ++++ db/schema.rb | 2 +- db/seeds.rb | 4 +- spec/models/account_spec.rb | 12 ++--- spec/services/fetch_remote_account_service_spec.rb | 1 - spec/services/fetch_resource_service_spec.rb | 4 +- spec/spec_helper.rb | 1 + 23 files changed, 141 insertions(+), 52 deletions(-) create mode 100644 app/controllers/instance_actors_controller.rb create mode 100644 db/migrate/20190715164535_add_instance_actor.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb index 52fb1dc1b..33bac9bbc 100644 --- a/app/controllers/about_controller.rb +++ b/app/controllers/about_controller.rb @@ -11,7 +11,9 @@ class AboutController < ApplicationController def show; end - def more; end + def more + flash.now[:notice] = I18n.t('about.instance_actor_flash') if params[:instance_actor] + end def terms; end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 26f3b1def..51e9764d4 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -91,7 +91,7 @@ class ApplicationController < ActionController::Base end def single_user_mode? - @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.exists? + @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.where('id > 0').exists? end def use_seamless_external_login? diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb index d1c525134..42493cd78 100644 --- a/app/controllers/home_controller.rb +++ b/app/controllers/home_controller.rb @@ -58,7 +58,7 @@ class HomeController < ApplicationController if request.path.start_with?('/web') new_user_session_path elsif single_user_mode? - short_account_path(Account.local.without_suspended.first) + short_account_path(Account.local.without_suspended.where('id > 0').first) else about_path end diff --git a/app/controllers/instance_actors_controller.rb b/app/controllers/instance_actors_controller.rb new file mode 100644 index 000000000..41f33602e --- /dev/null +++ b/app/controllers/instance_actors_controller.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class InstanceActorsController < ApplicationController + include AccountControllerConcern + + def show + expires_in 10.minutes, public: true + render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to + end + + private + + def set_account + @account = Account.find(-99) + end + + def restrict_fields_to + %i(id type preferred_username inbox public_key endpoints url manually_approves_followers) + end +end diff --git a/app/javascript/styles/mastodon/containers.scss b/app/javascript/styles/mastodon/containers.scss index 3564bf07b..2b6794ee2 100644 --- a/app/javascript/styles/mastodon/containers.scss +++ b/app/javascript/styles/mastodon/containers.scss @@ -145,6 +145,10 @@ min-height: 100%; } + .flash-message { + margin-bottom: 10px; + } + @media screen and (max-width: 738px) { grid-template-columns: minmax(0, 50%) minmax(0, 50%); diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb index 3eb88339a..28f1da19f 100644 --- a/app/lib/activitypub/activity/follow.rb +++ b/app/lib/activitypub/activity/follow.rb @@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account) - if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? + if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor? reject_follow_request!(target_account) return end diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 4d452f290..512272dbe 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -17,7 +17,7 @@ class ActivityPub::TagManager case target.object_type when :person - short_account_url(target) + target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? short_account_status_url(target.account, target) @@ -29,7 +29,7 @@ class ActivityPub::TagManager case target.object_type when :person - account_url(target) + target.instance_actor? ? instance_actor_url : account_url(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? account_status_url(target.account, target) @@ -119,6 +119,7 @@ class ActivityPub::TagManager def uri_to_local_id(uri, param = :id) path_params = Rails.application.routes.recognize_path(uri) + path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors' path_params[param] end diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb index a54a702a2..22d78874a 100644 --- a/app/lib/webfinger_resource.rb +++ b/app/lib/webfinger_resource.rb @@ -23,11 +23,17 @@ class WebfingerResource def username_from_url if account_show_page? path_params[:username] + elsif instance_actor_page? + Rails.configuration.x.local_domain else raise ActiveRecord::RecordNotFound end end + def instance_actor_page? + path_params[:controller] == 'instance_actors' + end + def account_show_page? path_params[:controller] == 'accounts' && path_params[:action] == 'show' end diff --git a/app/models/account.rb b/app/models/account.rb index adf4586fa..ccd116d6e 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -77,7 +77,7 @@ class Account < ApplicationRecord validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? } # Local user validations - validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? } + validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' } validates_with UniqueUsernameValidator, if: -> { local? && will_save_change_to_username? } validates_with UnreservedUsernameValidator, if: -> { local? && will_save_change_to_username? } validates :display_name, length: { maximum: 30 }, if: -> { local? && will_save_change_to_display_name? } @@ -139,6 +139,10 @@ class Account < ApplicationRecord %w(Application Service).include? actor_type end + def instance_actor? + id == -99 + end + alias bot bot? def bot=(val) @@ -498,7 +502,7 @@ class Account < ApplicationRecord end def generate_keys - return unless local? && !Rails.env.test? + return unless local? && private_key.blank? && public_key.blank? keypair = OpenSSL::PKey::RSA.new(2048) self.private_key = keypair.to_pem diff --git a/app/models/concerns/account_finder_concern.rb b/app/models/concerns/account_finder_concern.rb index ccd7bfa12..a54c2174d 100644 --- a/app/models/concerns/account_finder_concern.rb +++ b/app/models/concerns/account_finder_concern.rb @@ -13,7 +13,7 @@ module AccountFinderConcern end def representative - find_local(Setting.site_contact_username.strip.gsub(/\A@/, '')) || Account.local.without_suspended.first + Account.find(-99) end def find_local(username) diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index 0644219fb..0bd7aed2e 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -39,11 +39,17 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer delegate :moved?, to: :object def id - account_url(object) + object.instance_actor? ? instance_actor_url : account_url(object) end def type - object.bot? ? 'Service' : 'Person' + if object.instance_actor? + 'Application' + elsif object.bot? + 'Service' + else + 'Person' + end end def following @@ -55,7 +61,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer end def inbox - account_inbox_url(object) + object.instance_actor? ? instance_actor_inbox_url : account_inbox_url(object) end def outbox @@ -95,7 +101,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer end def url - short_account_url(object) + object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object) end def avatar_exists? diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb index f4af21551..008d0c182 100644 --- a/app/serializers/webfinger_serializer.rb +++ b/app/serializers/webfinger_serializer.rb @@ -10,15 +10,26 @@ class WebfingerSerializer < ActiveModel::Serializer end def aliases - [short_account_url(object), account_url(object)] + if object.instance_actor? + [instance_actor_url] + else + [short_account_url(object), account_url(object)] + end end def links - [ - { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) }, - { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') }, - { rel: 'self', type: 'application/activity+json', href: account_url(object) }, - { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" }, - ] + if object.instance_actor? + [ + { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) }, + { rel: 'self', type: 'application/activity+json', href: instance_actor_url }, + ] + else + [ + { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) }, + { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') }, + { rel: 'self', type: 'application/activity+json', href: account_url(object) }, + { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" }, + ] + end end end diff --git a/app/views/about/more.html.haml b/app/views/about/more.html.haml index b248ed1d2..21431ef8e 100644 --- a/app/views/about/more.html.haml +++ b/app/views/about/more.html.haml @@ -43,5 +43,7 @@ = mail_to @instance_presenter.site_contact_email, nil, title: @instance_presenter.site_contact_email .column-3 + = render 'application/flashes' + .box-widget .rich-formatting= @instance_presenter.site_extended_description.html_safe.presence || t('about.extended_description_html') diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby index ae80df9d2..f5a54052a 100644 --- a/app/views/well_known/webfinger/show.xml.ruby +++ b/app/views/well_known/webfinger/show.xml.ruby @@ -4,30 +4,47 @@ doc << Ox::Element.new('XRD').tap do |xrd| xrd['xmlns'] = 'http://docs.oasis-open.org/ns/xri/xrd-1.0' xrd << (Ox::Element.new('Subject') << @account.to_webfinger_s) - xrd << (Ox::Element.new('Alias') << short_account_url(@account)) - xrd << (Ox::Element.new('Alias') << account_url(@account)) - xrd << Ox::Element.new('Link').tap do |link| - link['rel'] = 'http://webfinger.net/rel/profile-page' - link['type'] = 'text/html' - link['href'] = short_account_url(@account) - end + if @account.instance_actor? + xrd << (Ox::Element.new('Alias') << instance_actor_url) - xrd << Ox::Element.new('Link').tap do |link| - link['rel'] = 'http://schemas.google.com/g/2010#updates-from' - link['type'] = 'application/atom+xml' - link['href'] = account_url(@account, format: 'atom') - end + xrd << Ox::Element.new('Link').tap do |link| + link['rel'] = 'http://webfinger.net/rel/profile-page' + link['type'] = 'text/html' + link['href'] = about_more_url(instance_actor: true) + end - xrd << Ox::Element.new('Link').tap do |link| - link['rel'] = 'self' - link['type'] = 'application/activity+json' - link['href'] = account_url(@account) - end + xrd << Ox::Element.new('Link').tap do |link| + link['rel'] = 'self' + link['type'] = 'application/activity+json' + link['href'] = instance_actor_url + end + else + xrd << (Ox::Element.new('Alias') << short_account_url(@account)) + xrd << (Ox::Element.new('Alias') << account_url(@account)) + + xrd << Ox::Element.new('Link').tap do |link| + link['rel'] = 'http://webfinger.net/rel/profile-page' + link['type'] = 'text/html' + link['href'] = short_account_url(@account) + end + + xrd << Ox::Element.new('Link').tap do |link| + link['rel'] = 'http://schemas.google.com/g/2010#updates-from' + link['type'] = 'application/atom+xml' + link['href'] = account_url(@account, format: 'atom') + end + + xrd << Ox::Element.new('Link').tap do |link| + link['rel'] = 'self' + link['type'] = 'application/activity+json' + link['href'] = account_url(@account) + end - xrd << Ox::Element.new('Link').tap do |link| - link['rel'] = 'http://ostatus.org/schema/1.0/subscribe' - link['template'] = "#{authorize_interaction_url}?acct={uri}" + xrd << Ox::Element.new('Link').tap do |link| + link['rel'] = 'http://ostatus.org/schema/1.0/subscribe' + link['template'] = "#{authorize_interaction_url}?acct={uri}" + end end end diff --git a/config/locales/en.yml b/config/locales/en.yml index 4e252945f..89c52b84a 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -24,6 +24,9 @@ en: generic_description: "%{domain} is one server in the network" get_apps: Try a mobile app hosted_on: Mastodon hosted on %{domain} + instance_actor_flash: | + This account is a virtual actor used to represent the server itself and not any individual user. + It is used for federation purposes and should not be blocked unless you want to block the whole instance, in which case you should use a domain block. learn_more: Learn more privacy_policy: Privacy policy see_whats_happening: See what's happening diff --git a/config/routes.rb b/config/routes.rb index 95f8a39ad..27b536641 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -28,6 +28,10 @@ Rails.application.routes.draw do get 'intent', to: 'intents#show' get 'custom.css', to: 'custom_css#show', as: :custom_css + resource :instance_actor, path: 'actor', only: [:show] do + resource :inbox, only: [:create], module: :activitypub + end + devise_scope :user do get '/invite/:invite_code', to: 'auth/registrations#new', as: :public_invite match '/auth/finish_signup' => 'auth/confirmations#finish_signup', via: [:get, :patch], as: :finish_signup diff --git a/db/migrate/20190715164535_add_instance_actor.rb b/db/migrate/20190715164535_add_instance_actor.rb new file mode 100644 index 000000000..a26d54949 --- /dev/null +++ b/db/migrate/20190715164535_add_instance_actor.rb @@ -0,0 +1,9 @@ +class AddInstanceActor < ActiveRecord::Migration[5.2] + def up + Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain) + end + + def down + Account.find_by(id: -99, actor_type: 'Application').destroy! + end +end diff --git a/db/schema.rb b/db/schema.rb index c7b6b9be6..a6a14827b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_06_233204) do +ActiveRecord::Schema.define(version: 2019_07_15_164535) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/db/seeds.rb b/db/seeds.rb index 9a6e9dd78..5f43fbac8 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,7 +1,9 @@ Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow') +domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain +Account.create!(id: -99, actor_type: 'Application', locked: true, username: domain) + if Rails.env.development? - domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain admin = Account.where(username: 'admin').first_or_initialize(username: 'admin') admin.save(validate: false) User.where(email: "admin@#{domain}").first_or_initialize(email: "admin@#{domain}", password: 'mastodonadmin', password_confirmation: 'mastodonadmin', confirmed_at: Time.now.utc, admin: true, account: admin, agreement: true, approved: true).save! diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index ce9ea250d..6495a6193 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -450,7 +450,7 @@ RSpec.describe Account, type: :model do describe '.domains' do it 'returns domains' do Fabricate(:account, domain: 'domain') - expect(Account.domains).to match_array(['domain']) + expect(Account.remote.domains).to match_array(['domain']) end end @@ -665,7 +665,7 @@ RSpec.describe Account, type: :model do { username: 'b', domain: 'b' }, ].map(&method(:Fabricate).curry(2).call(:account)) - expect(Account.alphabetic).to eq matches + expect(Account.where('id > 0').alphabetic).to eq matches end end @@ -732,7 +732,7 @@ RSpec.describe Account, type: :model do 2.times { Fabricate(:account, domain: 'example.com') } Fabricate(:account, domain: 'example2.com') - results = Account.by_domain_accounts + results = Account.where('id > 0').by_domain_accounts expect(results.length).to eq 2 expect(results.first.domain).to eq 'example.com' expect(results.first.accounts_count).to eq 2 @@ -745,7 +745,7 @@ RSpec.describe Account, type: :model do it 'returns an array of accounts who do not have a domain' do account_1 = Fabricate(:account, domain: nil) account_2 = Fabricate(:account, domain: 'example.com') - expect(Account.local).to match_array([account_1]) + expect(Account.where('id > 0').local).to match_array([account_1]) end end @@ -756,14 +756,14 @@ RSpec.describe Account, type: :model do matches[index] = Fabricate(:account, domain: matches[index]) end - expect(Account.partitioned).to match_array(matches) + expect(Account.where('id > 0').partitioned).to match_array(matches) end end describe 'recent' do it 'returns a relation of accounts sorted by recent creation' do matches = 2.times.map { Fabricate(:account) } - expect(Account.recent).to match_array(matches) + expect(Account.where('id > 0').recent).to match_array(matches) end end diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb index b37445861..ee7325be2 100644 --- a/spec/services/fetch_remote_account_service_spec.rb +++ b/spec/services/fetch_remote_account_service_spec.rb @@ -4,7 +4,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do let(:url) { 'https://example.com/alice' } let(:prefetched_body) { nil } let(:protocol) { :ostatus } - let!(:representative) { Fabricate(:account) } subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) } diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb index 98630966b..f836147d3 100644 --- a/spec/services/fetch_resource_service_spec.rb +++ b/spec/services/fetch_resource_service_spec.rb @@ -1,8 +1,6 @@ require 'rails_helper' RSpec.describe FetchResourceService, type: :service do - let!(:representative) { Fabricate(:account) } - describe '#call' do let(:url) { 'http://example.com' } @@ -60,7 +58,7 @@ RSpec.describe FetchResourceService, type: :service do it 'signs request' do subject - expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(representative) + '#main-key')}"/ })).to have_been_made + expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(Account.representative) + '#main-key')}"/ })).to have_been_made end context 'when content type is application/atom+xml' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 0cd1f91d0..45ba1bbd9 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -27,6 +27,7 @@ RSpec.configure do |config| end config.before :suite do + Rails.application.load_seed Chewy.strategy(:bypass) end -- cgit From f371b32137ccd7e74ca29d25af2072fb79654b15 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 28 Jul 2019 05:59:51 +0200 Subject: Change hashtags to preserve first-used casing (#11416) --- app/lib/activitypub/activity/create.rb | 9 ++---- app/models/tag.rb | 34 +++++++++++++++++++--- app/services/hashtag_query_service.rb | 4 +-- app/services/process_hashtags_service.rb | 4 +-- ...726175042_add_case_insensitive_index_to_tags.rb | 15 ++++++++++ db/schema.rb | 5 ++-- 6 files changed, 53 insertions(+), 18 deletions(-) create mode 100644 db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 56c24680a..000b77df5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -148,12 +148,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def process_hashtag(tag) return if tag['name'].blank? - hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase - hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag) - - return if @tags.include?(hashtag) - - @tags << hashtag + Tag.find_or_create_by_names(tag['name']) do |hashtag| + @tags << hashtag unless @tags.include?(hashtag) + end rescue ActiveRecord::RecordInvalid nil end diff --git a/app/models/tag.rb b/app/models/tag.rb index b371d59c1..972242064 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -20,7 +20,7 @@ class Tag < ApplicationRecord HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)' HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i - validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } + validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) } scope :hidden, -> { where(account_tag_stats: { hidden: true }) } @@ -64,22 +64,48 @@ class Tag < ApplicationRecord end class << self + def find_or_create_by_names(name_or_names) + Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name| + tag = matching_name(normalized_name).first || create(name: normalized_name) + + yield tag if block_given? + + tag + end + end + def search_for(term, limit = 5, offset = 0) - pattern = sanitize_sql_like(term.strip) + '%' + pattern = sanitize_sql_like(normalize(term.strip)) + '%' - Tag.where('lower(name) like lower(?)', pattern) + Tag.where(arel_table[:name].lower.matches(pattern.downcase)) .order(:name) .limit(limit) .offset(offset) end def find_normalized(name) - find_by(name: name.mb_chars.downcase.to_s) + matching_name(name).first end def find_normalized!(name) find_normalized(name) || raise(ActiveRecord::RecordNotFound) end + + def matching_name(name_or_names) + names = Array(name_or_names).map { |name| normalize(name).downcase } + + if names.size == 1 + where(arel_table[:name].lower.eq(names.first)) + else + where(arel_table[:name].lower.in(names)) + end + end + + private + + def normalize(str) + str.gsub(/\A#/, '').mb_chars.to_s + end end private diff --git a/app/services/hashtag_query_service.rb b/app/services/hashtag_query_service.rb index 5773d78c6..282821710 100644 --- a/app/services/hashtag_query_service.rb +++ b/app/services/hashtag_query_service.rb @@ -14,7 +14,7 @@ class HashtagQueryService < BaseService private - def tags_for(tags) - Tag.where(name: tags.map(&:downcase)) if tags.presence + def tags_for(names) + Tag.matching_name(names) if names.presence end end diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index b6974e598..e8e139b05 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -5,9 +5,7 @@ class ProcessHashtagsService < BaseService tags = Extractor.extract_hashtags(status.text) if status.local? records = [] - tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name| - tag = Tag.where(name: name).first_or_create(name: name) - + Tag.find_or_create_by_names(tags) do |tag| status.tags << tag records << tag diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb new file mode 100644 index 000000000..6fa8c0ec4 --- /dev/null +++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb @@ -0,0 +1,15 @@ +class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def up + safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' } + remove_index :tags, name: 'index_tags_on_name' + remove_index :tags, name: 'hashtag_search_index' + end + + def down + add_index :tags, :name, unique: true, algorithm: :concurrently + safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' } + remove_index :tags, name: 'index_tags_on_name_lower' + end +end diff --git a/db/schema.rb b/db/schema.rb index 6319dd932..1847305c7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_15_164535) do +ActiveRecord::Schema.define(version: 2019_07_26_175042) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -652,8 +652,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do t.string "name", default: "", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index" - t.index ["name"], name: "index_tags_on_name", unique: true + t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true end create_table "tombstones", force: :cascade do |t| -- cgit From 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 28 Jul 2019 17:30:12 +0200 Subject: Remove conversation URI (#11423) It is not part of ActivityPub and will free up a lot of space --- app/lib/activitypub/activity/create.rb | 11 ----------- app/models/conversation.rb | 7 ------- app/serializers/activitypub/note_serializer.rb | 15 ++------------- .../20190519130537_remove_boosts_widening_audience.rb | 2 ++ .../20190728084117_remove_uri_from_conversations.rb | 12 ++++++++++++ db/schema.rb | 4 +--- spec/models/conversation_spec.rb | 8 -------- 7 files changed, 17 insertions(+), 42 deletions(-) create mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 000b77df5..eb7b09e44 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -67,7 +67,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity sensitive: @object['sensitive'] || false, visibility: visibility_from_audience, thread: replied_to_status, - conversation: conversation_from_uri(@object['conversation']), media_attachment_ids: process_attachments.take(4).map(&:id), poll: process_poll, } @@ -262,16 +261,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil? end - def conversation_from_uri(uri) - return nil if uri.nil? - return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri) - begin - Conversation.find_or_create_by!(uri: uri) - rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique - retry - end - end - def visibility_from_audience if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public]) :public diff --git a/app/models/conversation.rb b/app/models/conversation.rb index 4dfaea889..37d233f32 100644 --- a/app/models/conversation.rb +++ b/app/models/conversation.rb @@ -4,17 +4,10 @@ # Table name: conversations # # id :bigint(8) not null, primary key -# uri :string # created_at :datetime not null # updated_at :datetime not null # class Conversation < ApplicationRecord - validates :uri, uniqueness: true, if: :uri? - has_many :statuses - - def local? - uri.nil? - end end diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 67f596e78..88b50b3ea 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -1,14 +1,13 @@ # frozen_string_literal: true class ActivityPub::NoteSerializer < ActivityPub::Serializer - context_extensions :atom_uri, :conversation, :sensitive, + context_extensions :atom_uri, :sensitive, :hashtag, :emoji, :focal_point, :blurhash attributes :id, :type, :summary, :in_reply_to, :published, :url, :attributed_to, :to, :cc, :sensitive, - :atom_uri, :in_reply_to_atom_uri, - :conversation + :atom_uri, :in_reply_to_atom_uri attribute :content attribute :content_map, if: :language? @@ -110,16 +109,6 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer OStatus::TagManager.instance.uri_for(object.thread) end - def conversation - return if object.conversation.nil? - - if object.conversation.uri? - object.conversation.uri - else - OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation') - end - end - def local? object.account.local? end diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb index d2d924239..b89efd989 100644 --- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb +++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2] disable_ddl_transaction! diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb new file mode 100644 index 000000000..b2b157ef7 --- /dev/null +++ b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RemoveUriFromConversations < ActiveRecord::Migration[5.2] + def up + safety_assured { remove_column :conversations, :uri, :string } + end + + def down + add_column :conversations, :uri, :string + add_index :conversations, :uri, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 1847305c7..a722bad36 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_26_175042) do +ActiveRecord::Schema.define(version: 2019_07_28_084117) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -203,10 +203,8 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do end create_table "conversations", force: :cascade do |t| - t.string "uri" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index ["uri"], name: "index_conversations_on_uri", unique: true end create_table "custom_emoji_categories", force: :cascade do |t| diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb index 8b5e4fdaf..b91d36169 100644 --- a/spec/models/conversation_spec.rb +++ b/spec/models/conversation_spec.rb @@ -1,13 +1,5 @@ require 'rails_helper' RSpec.describe Conversation, type: :model do - describe '#local?' do - it 'returns true when URI is nil' do - expect(Fabricate(:conversation).local?).to be true - end - it 'returns false when URI is not nil' do - expect(Fabricate(:conversation, uri: 'abc').local?).to be false - end - end end -- cgit From b9b0313c78ca4efcc8cda089c652be0b333aa16a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 28 Jul 2019 17:47:37 +0200 Subject: Revert "Remove conversation URI (#11423)" (#11424) This reverts commit 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449. --- app/lib/activitypub/activity/create.rb | 11 +++++++++++ app/models/conversation.rb | 7 +++++++ app/serializers/activitypub/note_serializer.rb | 15 +++++++++++++-- .../20190519130537_remove_boosts_widening_audience.rb | 2 -- .../20190728084117_remove_uri_from_conversations.rb | 12 ------------ db/schema.rb | 4 +++- spec/models/conversation_spec.rb | 8 ++++++++ 7 files changed, 42 insertions(+), 17 deletions(-) delete mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index eb7b09e44..000b77df5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -67,6 +67,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity sensitive: @object['sensitive'] || false, visibility: visibility_from_audience, thread: replied_to_status, + conversation: conversation_from_uri(@object['conversation']), media_attachment_ids: process_attachments.take(4).map(&:id), poll: process_poll, } @@ -261,6 +262,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil? end + def conversation_from_uri(uri) + return nil if uri.nil? + return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri) + begin + Conversation.find_or_create_by!(uri: uri) + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique + retry + end + end + def visibility_from_audience if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public]) :public diff --git a/app/models/conversation.rb b/app/models/conversation.rb index 37d233f32..4dfaea889 100644 --- a/app/models/conversation.rb +++ b/app/models/conversation.rb @@ -4,10 +4,17 @@ # Table name: conversations # # id :bigint(8) not null, primary key +# uri :string # created_at :datetime not null # updated_at :datetime not null # class Conversation < ApplicationRecord + validates :uri, uniqueness: true, if: :uri? + has_many :statuses + + def local? + uri.nil? + end end diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 88b50b3ea..67f596e78 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -1,13 +1,14 @@ # frozen_string_literal: true class ActivityPub::NoteSerializer < ActivityPub::Serializer - context_extensions :atom_uri, :sensitive, + context_extensions :atom_uri, :conversation, :sensitive, :hashtag, :emoji, :focal_point, :blurhash attributes :id, :type, :summary, :in_reply_to, :published, :url, :attributed_to, :to, :cc, :sensitive, - :atom_uri, :in_reply_to_atom_uri + :atom_uri, :in_reply_to_atom_uri, + :conversation attribute :content attribute :content_map, if: :language? @@ -109,6 +110,16 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer OStatus::TagManager.instance.uri_for(object.thread) end + def conversation + return if object.conversation.nil? + + if object.conversation.uri? + object.conversation.uri + else + OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation') + end + end + def local? object.account.local? end diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb index b89efd989..d2d924239 100644 --- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb +++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2] disable_ddl_transaction! diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb deleted file mode 100644 index b2b157ef7..000000000 --- a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -class RemoveUriFromConversations < ActiveRecord::Migration[5.2] - def up - safety_assured { remove_column :conversations, :uri, :string } - end - - def down - add_column :conversations, :uri, :string - add_index :conversations, :uri, unique: true - end -end diff --git a/db/schema.rb b/db/schema.rb index a722bad36..1847305c7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_28_084117) do +ActiveRecord::Schema.define(version: 2019_07_26_175042) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -203,8 +203,10 @@ ActiveRecord::Schema.define(version: 2019_07_28_084117) do end create_table "conversations", force: :cascade do |t| + t.string "uri" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.index ["uri"], name: "index_conversations_on_uri", unique: true end create_table "custom_emoji_categories", force: :cascade do |t| diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb index b91d36169..8b5e4fdaf 100644 --- a/spec/models/conversation_spec.rb +++ b/spec/models/conversation_spec.rb @@ -1,5 +1,13 @@ require 'rails_helper' RSpec.describe Conversation, type: :model do + describe '#local?' do + it 'returns true when URI is nil' do + expect(Fabricate(:conversation).local?).to be true + end + it 'returns false when URI is not nil' do + expect(Fabricate(:conversation, uri: 'abc').local?).to be false + end + end end -- cgit From 97192d9a77c0b4b68afe50d6a94d87110a8adbcd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Aug 2019 04:17:12 +0200 Subject: Fix remote and staff-removed statuses leaving media behind for a day (#11638) The reason for unattaching media instead of removing it is to support delete & redraft functionality, but remote or staff-removed statuses will never be redrafted, so the media should be deleted immediately --- app/controllers/api/v1/statuses_controller.rb | 2 +- app/lib/activitypub/activity/delete.rb | 2 +- app/models/form/status_batch.rb | 2 +- app/services/batched_remove_status_service.rb | 2 +- app/services/remove_status_service.rb | 12 ++++++++++++ app/workers/removal_worker.rb | 4 ++-- spec/controllers/admin/reported_statuses_controller_spec.rb | 2 +- spec/controllers/admin/statuses_controller_spec.rb | 2 +- spec/models/form/status_batch_spec.rb | 4 ++-- 9 files changed, 22 insertions(+), 10 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 71a505c26..39ca56482 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -53,7 +53,7 @@ class Api::V1::StatusesController < Api::BaseController @status = Status.where(account_id: current_user.account).find(params[:id]) authorize @status, :destroy? - RemovalWorker.perform_async(@status.id) + RemovalWorker.perform_async(@status.id, redraft: true) render json: @status, serializer: REST::StatusSerializer, source_requested: true end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 1f2b40c15..345060462 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -70,7 +70,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity end def delete_now! - RemoveStatusService.new.call(@status) + RemoveStatusService.new.call(@status, redraft: false) end def payload diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index 933dfdaca..831d8b7c5 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -34,7 +34,7 @@ class Form::StatusBatch def delete_statuses Status.where(id: status_ids).reorder(nil).find_each do |status| - RemovalWorker.perform_async(status.id) + RemovalWorker.perform_async(status.id, redraft: false) Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) log_action :destroy, status end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 6df8d4769..3638134be 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -8,7 +8,7 @@ class BatchedRemoveStatusService < BaseService # Dispatch Salmon deletes, unique per domain, of the deleted statuses, but only local ones # Remove statuses from home feeds # Push delete events to streaming API for home feeds and public feeds - # @param [Status] statuses A preferably batched array of statuses + # @param [Enumerable] statuses A preferably batched array of statuses # @param [Hash] options # @option [Boolean] :skip_side_effects def call(statuses, **options) diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 91c934181..685c1d4bf 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -4,6 +4,11 @@ class RemoveStatusService < BaseService include Redisable include Payloadable + # Delete a status + # @param [Status] status + # @param [Hash] options + # @option [Boolean] :redraft + # @options [Boolean] :original_removed def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @status = status @@ -24,6 +29,7 @@ class RemoveStatusService < BaseService remove_from_public remove_from_media if status.media_attachments.any? remove_from_spam_check + remove_media @status.destroy! else @@ -143,6 +149,12 @@ class RemoveStatusService < BaseService redis.publish('timeline:public:local:media', @payload) if @status.local? end + def remove_media + return if @options[:redraft] + + @status.media_attachments.destroy_all + end + def remove_from_spam_check redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id) end diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb index 19a660dd3..14423a4fb 100644 --- a/app/workers/removal_worker.rb +++ b/app/workers/removal_worker.rb @@ -3,8 +3,8 @@ class RemovalWorker include Sidekiq::Worker - def perform(status_id) - RemoveStatusService.new.call(Status.find(status_id)) + def perform(status_id, options = {}) + RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys) rescue ActiveRecord::RecordNotFound true end diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb index c358506d6..bd146b795 100644 --- a/spec/controllers/admin/reported_statuses_controller_spec.rb +++ b/spec/controllers/admin/reported_statuses_controller_spec.rb @@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 1a08c10b7..6b06343ef 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -65,7 +65,7 @@ describe Admin::StatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) end end diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb index 00c790a11..f9c58c90f 100644 --- a/spec/models/form/status_batch_spec.rb +++ b/spec/models/form/status_batch_spec.rb @@ -41,12 +41,12 @@ describe Form::StatusBatch do it 'call RemovalWorker' do form.save - expect(RemovalWorker).to have_received(:perform_async).with(status.id) + expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false) end it 'do not call RemovalWorker' do form.save - expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id) + expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false) end end end -- cgit From 031ca25014e0ba88d3dcc3086947b41449a672e2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 10 Sep 2019 15:29:12 +0200 Subject: Add retry for failed media downloads and `tootctl media refresh` (#11775) --- app/lib/activitypub/activity/create.rb | 21 +++++++------ app/models/concerns/remotable.rb | 12 ++++---- app/models/media_attachment.rb | 2 +- app/workers/redownload_media_worker.rb | 19 ++++++++++++ lib/mastodon/media_cli.rb | 54 ++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 app/workers/redownload_media_worker.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 000b77df5..dea7fd43c 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -189,22 +189,25 @@ class ActivityPub::Activity::Create < ActivityPub::Activity media_attachments = [] as_array(@object['attachment']).each do |attachment| - next if attachment['url'].blank? + next if attachment['url'].blank? || media_attachments.size >= 4 - href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) - media_attachments << media_attachment + begin + href = Addressable::URI.parse(attachment['url']).normalize.to_s + media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachments << media_attachment - next if unsupported_media_type?(attachment['mediaType']) || skip_download? + next if unsupported_media_type?(attachment['mediaType']) || skip_download? - media_attachment.file_remote_url = href - media_attachment.save + media_attachment.file_remote_url = href + media_attachment.save + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError + RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) + end end media_attachments rescue Addressable::URI::InvalidURIError => e - Rails.logger.debug e - + Rails.logger.debug "Invalid URL in attachment: #{e}" media_attachments end diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index 9372a963b..082302619 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -4,7 +4,7 @@ module Remotable extend ActiveSupport::Concern class_methods do - def remotable_attachment(attachment_name, limit) + def remotable_attachment(attachment_name, limit, suppress_errors: true) attribute_name = "#{attachment_name}_remote_url".to_sym method_name = "#{attribute_name}=".to_sym alt_method_name = "reset_#{attachment_name}!".to_sym @@ -22,7 +22,7 @@ module Remotable begin Request.new(:get, url).perform do |response| - next if response.code != 200 + raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code) content_type = parse_content_type(response.headers.get('content-type').last) extname = detect_extname_from_content_type(content_type) @@ -41,11 +41,11 @@ module Remotable self[attribute_name] = url if has_attribute?(attribute_name) end - rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e + Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" + raise e unless suppress_errors + rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" - nil - rescue Paperclip::Error, Mastodon::DimensionsValidationError => e - Rails.logger.debug "Error processing remote #{attachment_name}: #{e}" nil end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index b58025015..a2b73f150 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -118,7 +118,7 @@ class MediaAttachment < ApplicationRecord validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format? - remotable_attachment :file, VIDEO_LIMIT + remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false include Attachmentable diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb new file mode 100644 index 000000000..98e995918 --- /dev/null +++ b/app/workers/redownload_media_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RedownloadMediaWorker + include Sidekiq::Worker + include ExponentialBackoff + + sidekiq_options queue: 'pull', retry: 3 + + def perform(id) + media_attachment = MediaAttachment.find(id) + + return if media_attachment.remote_url.blank? + + media_attachment.reset_file! + media_attachment.save + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 0659b6b65..ec2f36c30 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -43,5 +43,59 @@ module Mastodon say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) end + + option :account, type: :string + option :domain, type: :string + option :status, type: :numeric + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, default: false, aliases: [:v] + option :dry_run, type: :boolean, default: false + desc 'refresh', 'Fetch remote media files' + long_desc <<-DESC + Re-downloads media attachments from other servers. You must specify the + source of media attachments with one of the following options: + + Use the --status option to download attachments from a specific status, + using the status local numeric ID. + + Use the --account option to download attachments from a specific account, + using username@domain handle of the account. + + Use the --domain option to download attachments from a specific domain. + DESC + def refresh + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + + if options[:status] + scope = MediaAttachment.where(status_id: options[:status]) + elsif options[:account] + username, domain = username.split('@') + account = Account.find_remote(username, domain) + + if account.nil? + say('No such account', :red) + exit(1) + end + + scope = MediaAttachment.where(account_id: account.id) + elsif options[:domain] + scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain])) + else + exit(1) + end + + processed, aggregate = parallelize_with_progress(scope) do |media_attachment| + next if media_attachment.remote_url.blank? + + unless options[:dry_run] + media_attachment.reset_file! + media_attachment.save + end + + media_attachment.file_file_size + end + + say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) + end end end -- cgit From c5d37f18cb3f4d6212fb8f3e1c4e1e027f677ec5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 11 Sep 2019 16:32:44 +0200 Subject: Change deletes to preserve soft-deleted statuses in unresolved reports (#11805) Change all account actions except "none" to resolve all unresolved reports Refactor `SuspendAccountService` to be more readable --- app/controllers/admin/accounts_controller.rb | 2 +- app/controllers/admin/report_notes_controller.rb | 9 ++-- .../api/v1/admin/accounts_controller.rb | 2 +- app/lib/activitypub/activity/delete.rb | 3 +- app/models/account.rb | 1 + app/models/admin/account_action.rb | 24 +++++++-- app/models/form/account_batch.rb | 2 +- app/models/form/status_batch.rb | 2 +- app/models/report.rb | 1 + app/models/status.rb | 4 ++ app/models/user.rb | 4 ++ app/services/block_domain_service.rb | 2 +- app/services/remove_status_service.rb | 7 +-- app/services/suspend_account_service.rb | 62 ++++++++++++++++------ app/services/unallow_domain_service.rb | 2 +- app/workers/admin/suspension_worker.rb | 2 +- lib/mastodon/accounts_cli.rb | 4 +- lib/mastodon/domains_cli.rb | 2 +- .../admin/reported_statuses_controller_spec.rb | 2 +- spec/controllers/admin/statuses_controller_spec.rb | 2 +- spec/models/form/status_batch_spec.rb | 4 +- 21 files changed, 98 insertions(+), 45 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb index 2fa1dfe5f..68b6352f8 100644 --- a/app/controllers/admin/accounts_controller.rb +++ b/app/controllers/admin/accounts_controller.rb @@ -41,7 +41,7 @@ module Admin def reject authorize @account.user, :reject? - SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true) + SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false) redirect_to admin_pending_accounts_path end diff --git a/app/controllers/admin/report_notes_controller.rb b/app/controllers/admin/report_notes_controller.rb index bcb3f2026..b816c5b5d 100644 --- a/app/controllers/admin/report_notes_controller.rb +++ b/app/controllers/admin/report_notes_controller.rb @@ -5,10 +5,10 @@ module Admin before_action :set_report_note, only: [:destroy] def create - authorize ReportNote, :create? + authorize :report_note, :create? @report_note = current_account.report_notes.new(resource_params) - @report = @report_note.report + @report = @report_note.report if @report_note.save if params[:create_and_resolve] @@ -26,9 +26,8 @@ module Admin redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg') else - @report_notes = @report.notes.latest - @report_history = @report.history - @form = Form::StatusBatch.new + @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at) + @form = Form::StatusBatch.new render template: 'admin/reports/show' end diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb index c306180ca..c35ea5ab2 100644 --- a/app/controllers/api/v1/admin/accounts_controller.rb +++ b/app/controllers/api/v1/admin/accounts_controller.rb @@ -58,7 +58,7 @@ class Api::V1::Admin::AccountsController < Api::BaseController def reject authorize @account.user, :reject? - SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true) + SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false) render json: @account, serializer: REST::Admin::AccountSerializer end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 345060462..dc9ff580c 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -13,8 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def delete_person lock_or_return("delete_in_progress:#{@account.id}") do - SuspendAccountService.new.call(@account) - @account.destroy! + SuspendAccountService.new.call(@account, reserve_username: false) end end diff --git a/app/models/account.rb b/app/models/account.rb index 8c9388b95..55fe53fae 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -115,6 +115,7 @@ class Account < ApplicationRecord :approved?, :pending?, :disabled?, + :unconfirmed_or_pending?, :role, :admin?, :moderator?, diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index c7da8b52c..b30a82369 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -83,19 +83,23 @@ class Admin::AccountAction # A log entry is only interesting if the warning contains # custom text from someone. Otherwise it's just noise. + log_action(:create, warning) if warning.text.present? end def process_reports! - return if report_id.blank? + # If we're doing "mark as resolved" on a single report, + # then we want to keep other reports open in case they + # contain new actionable information. + # + # Otherwise, we will mark all unresolved reports about + # the account as resolved. - authorize(report, :update?) + reports.each { |report| authorize(report, :update?) } - if type == 'none' + reports.each do |report| log_action(:resolve, report) report.resolve!(current_account) - else - Report.where(target_account: target_account).unresolved.update_all(action_taken: true, action_taken_by_account_id: current_account.id) end end @@ -141,6 +145,16 @@ class Admin::AccountAction @report.status_ids if @report && include_statuses end + def reports + @reports ||= begin + if type == 'none' && with_report? + [report] + else + Report.where(target_account: target_account).unresolved + end + end + end + def warning_preset @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present? end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index f1b7a4566..0b285fde9 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -69,6 +69,6 @@ class Form::AccountBatch records = accounts.includes(:user) records.each { |account| authorize(account.user, :reject?) } - .each { |account| SuspendAccountService.new.call(account, including_user: true, destroy: true, skip_distribution: true) } + .each { |account| SuspendAccountService.new.call(account, reserve_email: false, reserve_username: false) } end end diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index e09cc2594..c4943a7ea 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -35,7 +35,7 @@ class Form::StatusBatch def delete_statuses Status.where(id: status_ids).reorder(nil).find_each do |status| status.discard - RemovalWorker.perform_async(status.id, redraft: false) + RemovalWorker.perform_async(status.id, immediate: true) Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) log_action :destroy, status end diff --git a/app/models/report.rb b/app/models/report.rb index 1e707ff1c..fb2e040ee 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -59,6 +59,7 @@ class Report < ApplicationRecord end def resolve!(acting_account) + RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] } update!(action_taken: true, action_taken_by_account_id: acting_account.id) end diff --git a/app/models/status.rb b/app/models/status.rb index 9cfaddcec..471bb03b4 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -214,6 +214,10 @@ class Status < ApplicationRecord !sensitive? && with_media? end + def reported? + @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists? + end + def emojis return @emojis if defined?(@emojis) diff --git a/app/models/user.rb b/app/models/user.rb index 95f1d8fc5..78b82a68f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -171,6 +171,10 @@ class User < ApplicationRecord confirmed? && approved? && !disabled? && !account.suspended? end + def unconfirmed_or_pending? + !(confirmed? && approved?) + end + def inactive_message !approved? ? :pending : super end diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb index 0ec6be503..ae461abf2 100644 --- a/app/services/block_domain_service.rb +++ b/app/services/block_domain_service.rb @@ -53,7 +53,7 @@ class BlockDomainService < BaseService def suspend_accounts! blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account| - SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at) + SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at) end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 685c1d4bf..f9352ed3d 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -8,7 +8,8 @@ class RemoveStatusService < BaseService # @param [Status] status # @param [Hash] options # @option [Boolean] :redraft - # @options [Boolean] :original_removed + # @option [Boolean] :immediate + # @option [Boolean] :original_removed def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @status = status @@ -31,7 +32,7 @@ class RemoveStatusService < BaseService remove_from_spam_check remove_media - @status.destroy! + @status.destroy! if @options[:immediate] || !@status.reported? else raise Mastodon::RaceConditionError end @@ -150,7 +151,7 @@ class RemoveStatusService < BaseService end def remove_media - return if @options[:redraft] + return if @options[:redraft] || (!@options[:immediate] && @status.reported?) @status.media_attachments.destroy_all end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 85da7e921..ecc893931 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -15,7 +15,6 @@ class SuspendAccountService < BaseService favourites follow_requests list_accounts - media_attachments mute_relationships muted_by_relationships notifications @@ -32,14 +31,26 @@ class SuspendAccountService < BaseService targeted_reports ).freeze - # Suspend an account and remove as much of its data as possible + # Suspend or remove an account and remove as much of its data + # as possible. If it's a local account and it has not been confirmed + # or never been approved, then side effects are skipped and both + # the user and account records are removed fully. Otherwise, + # it is controlled by options. # @param [Account] # @param [Hash] options - # @option [Boolean] :including_user Remove the user record as well - # @option [Boolean] :destroy Remove the account record instead of suspending + # @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts + # @option [Boolean] :reserve_username Keep account record + # @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads + # @option [Time] :suspended_at Only applicable when :reserve_username is true def call(account, **options) @account = account - @options = options + @options = { reserve_username: true, reserve_email: true }.merge(options) + + if @account.local? && @account.user_unconfirmed_or_pending? + @options[:reserve_email] = false + @options[:reserve_username] = false + @options[:skip_side_effects] = true + end reject_follows! purge_user! @@ -60,27 +71,39 @@ class SuspendAccountService < BaseService def purge_user! return if !@account.local? || @account.user.nil? - if @options[:including_user] - @options[:destroy] = true if !@account.user_confirmed? || @account.user_pending? - @account.user.destroy - else + if @options[:reserve_email] @account.user.disable! @account.user.invites.where(uses: 0).destroy_all + else + @account.user.destroy end end def purge_content! - distribute_delete_actor! if @account.local? && !@options[:skip_distribution] + distribute_delete_actor! if @account.local? && !@options[:skip_side_effects] @account.statuses.reorder(nil).find_in_batches do |statuses| - BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy]) + statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username] + BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects]) + end + + @account.media_attachments.reorder(nil).find_each do |media_attachment| + next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id) + + media_attachment.destroy + end + + @account.polls.reorder(nil).find_each do |poll| + next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id) + + poll.destroy end associations_for_destruction.each do |association_name| destroy_all(@account.public_send(association_name)) end - @account.destroy if @options[:destroy] + @account.destroy unless @options[:reserve_username] end def purge_profile! @@ -88,11 +111,13 @@ class SuspendAccountService < BaseService # there is no point wasting time updating # its values first - return if @options[:destroy] + return unless @options[:reserve_username] @account.silenced_at = nil @account.suspended_at = @options[:suspended_at] || Time.now.utc @account.locked = false + @account.memorial = false + @account.discoverable = false @account.display_name = '' @account.note = '' @account.fields = [] @@ -100,6 +125,7 @@ class SuspendAccountService < BaseService @account.followers_count = 0 @account.following_count = 0 @account.moved_to_account = nil + @account.trust_level = :untrusted @account.avatar.destroy @account.header.destroy @account.save! @@ -135,11 +161,15 @@ class SuspendAccountService < BaseService Account.inboxes - delivery_inboxes end + def reported_status_ids + @reported_status_ids ||= Report.where(target_account: @account).unresolved.pluck(:status_ids).flatten.uniq + end + def associations_for_destruction - if @options[:destroy] - ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY - else + if @options[:reserve_username] ASSOCIATIONS_ON_SUSPEND + else + ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY end end end diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb index d4387c1a1..bd1ad328d 100644 --- a/app/services/unallow_domain_service.rb +++ b/app/services/unallow_domain_service.rb @@ -3,7 +3,7 @@ class UnallowDomainService < BaseService def call(domain_allow) Account.where(domain: domain_allow.domain).find_each do |account| - SuspendAccountService.new.call(account, destroy: true) + SuspendAccountService.new.call(account, reserve_username: false) end domain_allow.destroy diff --git a/app/workers/admin/suspension_worker.rb b/app/workers/admin/suspension_worker.rb index ae8b24d8c..83c815efd 100644 --- a/app/workers/admin/suspension_worker.rb +++ b/app/workers/admin/suspension_worker.rb @@ -6,6 +6,6 @@ class Admin::SuspensionWorker sidekiq_options queue: 'pull' def perform(account_id, remove_user = false) - SuspendAccountService.new.call(Account.find(account_id), including_user: remove_user) + SuspendAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: !remove_user) end end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index b16bf2e38..a09a6ab04 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -185,7 +185,7 @@ module Mastodon end say("Deleting user with #{account.statuses_count} statuses, this might take a while...") - SuspendAccountService.new.call(account, including_user: true) + SuspendAccountService.new.call(account, reserve_email: false) say('OK', :green) end @@ -239,7 +239,7 @@ module Mastodon end if [404, 410].include?(code) - SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + SuspendAccountService.new.call(account, reserve_username: false) unless options[:dry_run] 1 else # Touch account even during dry run to avoid getting the account into the window again diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index c612c2d72..8e52de1c3 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -42,7 +42,7 @@ module Mastodon end processed, = parallelize_with_progress(scope) do |account| - SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run] end DomainBlock.where(domain: domain).destroy_all unless options[:dry_run] diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb index bd146b795..2a1598123 100644 --- a/spec/controllers/admin/reported_statuses_controller_spec.rb +++ b/spec/controllers/admin/reported_statuses_controller_spec.rb @@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true) end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 6b06343ef..d9690d83f 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -65,7 +65,7 @@ describe Admin::StatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true) end end diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb index f9c58c90f..68d84a737 100644 --- a/spec/models/form/status_batch_spec.rb +++ b/spec/models/form/status_batch_spec.rb @@ -41,12 +41,12 @@ describe Form::StatusBatch do it 'call RemovalWorker' do form.save - expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status.id, immediate: true) end it 'do not call RemovalWorker' do form.save - expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false) + expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, immediate: true) end end end -- cgit From 38dc51b2d68e3e03f429419f7318e868fa24c49d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 17 Sep 2019 08:44:45 +0200 Subject: Fix Move handler queuing jobs that will fail if account is suspended (#11864) Don't put Move handler on cooldown if it didn't run. Skip unmerging from timelines to save unnecessary work. --- app/lib/activitypub/activity/move.rb | 11 +++++++++-- app/services/unfollow_service.rb | 11 +++++++++-- app/workers/unfollow_follow_worker.rb | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index d7a5f595c..6c6a2b967 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -10,10 +10,13 @@ class ActivityPub::Activity::Move < ActivityPub::Activity target_account = ActivityPub::FetchRemoteAccountService.new.call(target_uri) - return if target_account.nil? || !target_account.also_known_as.include?(origin_account.uri) + if target_account.nil? || target_account.suspended? || !target_account.also_known_as.include?(origin_account.uri) + unmark_as_processing! + return + end # In case for some reason we didn't have a redirect for the profile already, set it - origin_account.update(moved_to_account: target_account) if origin_account.moved_to_account_id.nil? + origin_account.update(moved_to_account: target_account) # Initiate a re-follow for each follower origin_account.followers.local.select(:id).find_in_batches do |follower_accounts| @@ -40,4 +43,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity def mark_as_processing! redis.setex("move_in_progress:#{@account.id}", PROCESSING_COOLDOWN, true) end + + def unmark_as_processing! + redis.del("move_in_progress:#{@account.id}") + end end diff --git a/app/services/unfollow_service.rb b/app/services/unfollow_service.rb index b7033d7eb..151f3674f 100644 --- a/app/services/unfollow_service.rb +++ b/app/services/unfollow_service.rb @@ -6,9 +6,12 @@ class UnfollowService < BaseService # Unfollow and notify the remote user # @param [Account] source_account Where to unfollow from # @param [Account] target_account Which to unfollow - def call(source_account, target_account) + # @param [Hash] options + # @option [Boolean] :skip_unmerge + def call(source_account, target_account, options = {}) @source_account = source_account @target_account = target_account + @options = options unfollow! || undo_follow_request! end @@ -21,9 +24,11 @@ class UnfollowService < BaseService return unless follow follow.destroy! + create_notification(follow) if !@target_account.local? && @target_account.activitypub? create_reject_notification(follow) if @target_account.local? && !@source_account.local? && @source_account.activitypub? - UnmergeWorker.perform_async(@target_account.id, @source_account.id) + UnmergeWorker.perform_async(@target_account.id, @source_account.id) unless @options[:skip_unmerge] + follow end @@ -33,7 +38,9 @@ class UnfollowService < BaseService return unless follow_request follow_request.destroy! + create_notification(follow_request) unless @target_account.local? + follow_request end diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb index 50d3bf034..95549e107 100644 --- a/app/workers/unfollow_follow_worker.rb +++ b/app/workers/unfollow_follow_worker.rb @@ -11,7 +11,7 @@ class UnfollowFollowWorker new_target_account = Account.find(new_target_account_id) FollowService.new.call(follower_account, new_target_account) - UnfollowService.new.call(follower_account, old_target_account) + UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true) rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError true end -- cgit From 4f6af87906175d9ea802ef0c6f050388eac890fa Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 18 Sep 2019 12:53:13 +0200 Subject: Change spam check to apply to local accounts and add a threshold (#11806) Instead of detecting spam on first duplicate message, add a threshold of 5 such messages to reduce false positives --- app/lib/activitypub/activity/create.rb | 10 +------ app/lib/spam_check.rb | 46 ++++++++++++++++++++++++++------ app/services/process_mentions_service.rb | 5 ++++ spec/lib/spam_check_spec.rb | 34 ++++++++++++++--------- 4 files changed, 66 insertions(+), 29 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index dea7fd43c..e69193b71 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -408,15 +408,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def check_for_spam - spam_check = SpamCheck.new(@status) - - return if spam_check.skip? - - if spam_check.spam? - spam_check.flag! - else - spam_check.remember! - end + SpamCheck.perform(@status) end def forward_for_reply diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb index 0cf1b8790..441697364 100644 --- a/app/lib/spam_check.rb +++ b/app/lib/spam_check.rb @@ -4,9 +4,25 @@ class SpamCheck include Redisable include ActionView::Helpers::TextHelper + # Threshold over which two Nilsimsa values are considered + # to refer to the same text NILSIMSA_COMPARE_THRESHOLD = 95 - NILSIMSA_MIN_SIZE = 10 - EXPIRE_SET_AFTER = 1.week.seconds + + # Nilsimsa doesn't work well on small inputs, so below + # this size, we check only for exact matches with MD5 + NILSIMSA_MIN_SIZE = 10 + + # How long to keep the trail of digests between updates, + # there is no reason to store it forever + EXPIRE_SET_AFTER = 1.week.seconds + + # How many digests to keep in an account's trail. If it's + # too small, spam could rotate around different message templates + MAX_TRAIL_SIZE = 10 + + # How many detected duplicates to allow through before + # considering the message as spam + THRESHOLD = 5 def initialize(status) @account = status.account @@ -21,9 +37,9 @@ class SpamCheck if insufficient_data? false elsif nilsimsa? - any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } + digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } else - any_other_digest?('md5') { |_, other_digest| other_digest == digest } + digests_over_threshold?('md5') { |_, other_digest| other_digest == digest } end end @@ -38,7 +54,7 @@ class SpamCheck # get the correct status ID back, we have to save it in the string value redis.zadd(redis_key, @status.id, digest_with_algorithm) - redis.zremrangebyrank(redis_key, '0', '-10') + redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1)) redis.expire(redis_key, EXPIRE_SET_AFTER) end @@ -78,6 +94,20 @@ class SpamCheck end end + class << self + def perform(status) + spam_check = new(status) + + return if spam_check.skip? + + if spam_check.spam? + spam_check.flag! + else + spam_check.remember! + end + end + end + private def disabled? @@ -149,14 +179,14 @@ class SpamCheck redis.zrange(redis_key, 0, -1) end - def any_other_digest?(filter_algorithm) - other_digests.any? do |record| + def digests_over_threshold?(filter_algorithm) + other_digests.select do |record| algorithm, other_digest, status_id = record.split(':') next unless algorithm == filter_algorithm yield algorithm, other_digest, status_id - end + end.size >= THRESHOLD end def matching_status_ids diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index 90dca9740..2f7a9e985 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -33,6 +33,7 @@ class ProcessMentionsService < BaseService end status.save! + check_for_spam(status) mentions.each { |mention| create_notification(mention) } end @@ -61,4 +62,8 @@ class ProcessMentionsService < BaseService def resolve_account_service ResolveAccountService.new end + + def check_for_spam(status) + SpamCheck.perform(status) + end end diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb index 9e0989216..4cae46111 100644 --- a/spec/lib/spam_check_spec.rb +++ b/spec/lib/spam_check_spec.rb @@ -86,23 +86,33 @@ RSpec.describe SpamCheck do end it 'returns true for duplicate statuses to the same recipient' do - status1 = status_with_html('@alice Hello') - described_class.new(status1).remember! + described_class::THRESHOLD.times do + status1 = status_with_html('@alice Hello') + described_class.new(status1).remember! + end + status2 = status_with_html('@alice Hello') expect(described_class.new(status2).spam?).to be true end it 'returns true for duplicate statuses to different recipients' do - status1 = status_with_html('@alice Hello') - described_class.new(status1).remember! + described_class::THRESHOLD.times do + status1 = status_with_html('@alice Hello') + described_class.new(status1).remember! + end + status2 = status_with_html('@bob Hello') expect(described_class.new(status2).spam?).to be true end it 'returns true for nearly identical statuses with random numbers' do source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.' - status1 = status_with_html('@alice ' + source_text + ' 1234') - described_class.new(status1).remember! + + described_class::THRESHOLD.times do + status1 = status_with_html('@alice ' + source_text + ' 1234') + described_class.new(status1).remember! + end + status2 = status_with_html('@bob ' + source_text + ' 9568') expect(described_class.new(status2).spam?).to be true end @@ -140,9 +150,9 @@ RSpec.describe SpamCheck do let(:redis_key) { spam_check.send(:redis_key) } it 'remembers' do - expect do - spam_check.remember! - end.to change { Redis.current.exists(redis_key) }.from(false).to(true) + expect(Redis.current.exists(redis_key)).to be true + spam_check.remember! + expect(Redis.current.exists(redis_key)).to be true end end @@ -156,9 +166,9 @@ RSpec.describe SpamCheck do end it 'resets' do - expect do - spam_check.reset! - end.to change { Redis.current.exists(redis_key) }.from(true).to(false) + expect(Redis.current.exists(redis_key)).to be true + spam_check.reset! + expect(Redis.current.exists(redis_key)).to be false end end -- cgit From 18b451c0e6cf6a927a22084f94b423982de0ee8b Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 27 Sep 2019 21:13:51 +0200 Subject: Change silences to always require approval on follow (#11975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change silenced accounts to require approval on follow * Also require approval for follows by people explicitly muted by target accounts * Do not auto-accept silenced or muted accounts when switching from locked to unlocked * Add `follow_requests_count` to verify_credentials * Show “Follow requests” menu item if needed even if account is locked * Add tests * Correctly reflect that follow requests weren't auto-accepted when local account is silenced * Accept follow requests from user-muted accounts to avoid leaking mutes --- app/controllers/api/v1/accounts_controller.rb | 2 +- .../mastodon/features/getting_started/index.js | 8 ++--- app/lib/activitypub/activity/follow.rb | 2 +- .../rest/credential_account_serializer.rb | 1 + app/services/follow_service.rb | 2 +- app/services/update_account_service.rb | 4 ++- spec/lib/activitypub/activity/follow_spec.rb | 30 +++++++++++++++++ spec/services/follow_service_spec.rb | 27 +++++++++++++++ spec/services/update_account_service_spec.rb | 38 ++++++++++++++++++++++ 9 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 spec/services/update_account_service_spec.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb index b306e8e8c..c12e1c12e 100644 --- a/app/controllers/api/v1/accounts_controller.rb +++ b/app/controllers/api/v1/accounts_controller.rb @@ -33,7 +33,7 @@ class Api::V1::AccountsController < Api::BaseController def follow FollowService.new.call(current_user.account, @account, reblogs: truthy_param?(:reblogs)) - options = @account.locked? ? {} : { following_map: { @account.id => { reblogs: truthy_param?(:reblogs) } }, requested_map: { @account.id => false } } + options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: truthy_param?(:reblogs) } }, requested_map: { @account.id => false } } render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(options) end diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js index f6d90580b..67ec7665b 100644 --- a/app/javascript/mastodon/features/getting_started/index.js +++ b/app/javascript/mastodon/features/getting_started/index.js @@ -77,16 +77,14 @@ class GettingStarted extends ImmutablePureComponent { }; componentDidMount () { - const { myAccount, fetchFollowRequests, multiColumn } = this.props; + const { fetchFollowRequests, multiColumn } = this.props; if (!multiColumn && window.innerWidth >= NAVIGATION_PANEL_BREAKPOINT) { this.context.router.history.replace('/timelines/home'); return; } - if (myAccount.get('locked')) { - fetchFollowRequests(); - } + fetchFollowRequests(); } render () { @@ -134,7 +132,7 @@ class GettingStarted extends ImmutablePureComponent { height += 48*3; - if (myAccount.get('locked')) { + if (myAccount.get('locked') || unreadFollowRequests > 0) { navItems.push(); height += 48; } diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb index 28f1da19f..ec92f4255 100644 --- a/app/lib/activitypub/activity/follow.rb +++ b/app/lib/activitypub/activity/follow.rb @@ -21,7 +21,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity follow_request = FollowRequest.create!(account: @account, target_account: target_account, uri: @json['id']) - if target_account.locked? + if target_account.locked? || @account.silenced? NotifyService.new.call(target_account, follow_request) else AuthorizeFollowService.new.call(@account, target_account) diff --git a/app/serializers/rest/credential_account_serializer.rb b/app/serializers/rest/credential_account_serializer.rb index fb195eb07..be0d763dc 100644 --- a/app/serializers/rest/credential_account_serializer.rb +++ b/app/serializers/rest/credential_account_serializer.rb @@ -12,6 +12,7 @@ class REST::CredentialAccountSerializer < REST::AccountSerializer language: user.setting_default_language, note: object.note, fields: object.fields.map(&:to_h), + follow_requests_count: FollowRequest.where(target_account: object).limit(40).count, } end end diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index 101acdaf9..1941c2e2d 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -30,7 +30,7 @@ class FollowService < BaseService ActivityTracker.increment('activity:interactions') - if target_account.locked? || target_account.activitypub? + if target_account.locked? || source_account.silenced? || target_account.activitypub? request_follow(source_account, target_account, reblogs: reblogs) elsif target_account.local? direct_follow(source_account, target_account, reblogs: reblogs) diff --git a/app/services/update_account_service.rb b/app/services/update_account_service.rb index 01756a73d..ebf24be37 100644 --- a/app/services/update_account_service.rb +++ b/app/services/update_account_service.rb @@ -20,7 +20,9 @@ class UpdateAccountService < BaseService private def authorize_all_follow_requests(account) - AuthorizeFollowWorker.push_bulk(FollowRequest.where(target_account: account).select(:account_id, :target_account_id)) do |req| + follow_requests = FollowRequest.where(target_account: account) + follow_requests = follow_requests.select { |req| !req.account.silenced? } + AuthorizeFollowWorker.push_bulk(follow_requests) do |req| [req.account_id, req.target_account_id] end end diff --git a/spec/lib/activitypub/activity/follow_spec.rb b/spec/lib/activitypub/activity/follow_spec.rb index 6bbacdbe6..05112cc18 100644 --- a/spec/lib/activitypub/activity/follow_spec.rb +++ b/spec/lib/activitypub/activity/follow_spec.rb @@ -31,6 +31,36 @@ RSpec.describe ActivityPub::Activity::Follow do end end + context 'silenced account following an unlocked account' do + before do + sender.touch(:silenced_at) + subject.perform + end + + it 'does not create a follow from sender to recipient' do + expect(sender.following?(recipient)).to be false + end + + it 'creates a follow request' do + expect(sender.requested?(recipient)).to be true + end + end + + context 'unlocked account muting the sender' do + before do + recipient.mute!(sender) + subject.perform + end + + it 'creates a follow from sender to recipient' do + expect(sender.following?(recipient)).to be true + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end + end + context 'locked account' do before do recipient.update(locked: true) diff --git a/spec/services/follow_service_spec.rb b/spec/services/follow_service_spec.rb index 86c85293e..ae863a9f0 100644 --- a/spec/services/follow_service_spec.rb +++ b/spec/services/follow_service_spec.rb @@ -30,6 +30,33 @@ RSpec.describe FollowService, type: :service do end end + describe 'unlocked account, from silenced account' do + let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + + before do + sender.touch(:silenced_at) + subject.call(sender, bob.acct) + end + + it 'creates a follow request with reblogs' do + expect(FollowRequest.find_by(account: sender, target_account: bob, show_reblogs: true)).to_not be_nil + end + end + + describe 'unlocked account, from a muted account' do + let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + + before do + bob.mute!(sender) + subject.call(sender, bob.acct) + end + + it 'creates a following relation with reblogs' do + expect(sender.following?(bob)).to be true + expect(sender.muting_reblogs?(bob)).to be false + end + end + describe 'unlocked account' do let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } diff --git a/spec/services/update_account_service_spec.rb b/spec/services/update_account_service_spec.rb new file mode 100644 index 000000000..960b26891 --- /dev/null +++ b/spec/services/update_account_service_spec.rb @@ -0,0 +1,38 @@ +require 'rails_helper' + +RSpec.describe UpdateAccountService, type: :service do + subject { UpdateAccountService.new } + + describe 'switching form locked to unlocked accounts' do + let(:account) { Fabricate(:account, locked: true) } + let(:alice) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account } + let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + let(:eve) { Fabricate(:user, email: 'eve@example.com', account: Fabricate(:account, username: 'eve')).account } + + before do + bob.touch(:silenced_at) + account.mute!(eve) + + FollowService.new.call(alice, account) + FollowService.new.call(bob, account) + FollowService.new.call(eve, account) + + subject.call(account, { locked: false }) + end + + it 'auto-accepts pending follow requests' do + expect(alice.following?(account)).to be true + expect(alice.requested?(account)).to be false + end + + it 'does not auto-accept pending follow requests from silenced users' do + expect(bob.following?(account)).to be false + expect(bob.requested?(account)).to be true + end + + it 'auto-accepts pending follow requests from muted users so as to not leak mute' do + expect(eve.following?(account)).to be true + expect(eve.requested?(account)).to be false + end + end +end -- cgit From 368a87755b4b12c37deb415e10e03c709012f698 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 29 Sep 2019 16:23:13 +0200 Subject: Fix account migration not affecting followers on origin server (#11980) --- app/controllers/settings/migrations_controller.rb | 4 +-- app/lib/activitypub/activity/move.rb | 6 +---- app/services/move_service.rb | 32 ++++++++++++++++++++++ app/workers/move_worker.rb | 33 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 app/services/move_service.rb create mode 100644 app/workers/move_worker.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb index 00bde1d61..68304bb51 100644 --- a/app/controllers/settings/migrations_controller.rb +++ b/app/controllers/settings/migrations_controller.rb @@ -18,9 +18,7 @@ class Settings::MigrationsController < Settings::BaseController @migration = current_account.migrations.build(resource_params) if @migration.save_with_challenge(current_user) - current_account.update!(moved_to_account: @migration.target_account) - ActivityPub::UpdateDistributionWorker.perform_async(current_account.id) - ActivityPub::MoveDistributionWorker.perform_async(@migration.id) + MoveService.new.call(@migration) redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct) else render :show diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 6c6a2b967..12bb82d25 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -19,11 +19,7 @@ class ActivityPub::Activity::Move < ActivityPub::Activity origin_account.update(moved_to_account: target_account) # Initiate a re-follow for each follower - origin_account.followers.local.select(:id).find_in_batches do |follower_accounts| - UnfollowFollowWorker.push_bulk(follower_accounts.map(&:id)) do |follower_account_id| - [follower_account_id, origin_account.id, target_account.id] - end - end + MoveWorker.perform_async(origin_account.id, target_account.id) end private diff --git a/app/services/move_service.rb b/app/services/move_service.rb new file mode 100644 index 000000000..da0c62c4e --- /dev/null +++ b/app/services/move_service.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class MoveService < BaseService + def call(migration) + @migration = migration + @source_account = migration.account + @target_account = migration.target_account + + update_redirect! + process_local_relationships! + distribute_update! + distribute_move! + end + + private + + def update_redirect! + @source_account.update!(moved_to_account: @target_account) + end + + def process_local_relationships! + MoveWorker.perform_async(@source_account.id, @target_account.id) + end + + def distribute_update! + ActivityPub::UpdateDistributionWorker.perform_async(@source_account.id) + end + + def distribute_move! + ActivityPub::MoveDistributionWorker.perform_async(@migration.id) + end +end diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb new file mode 100644 index 000000000..22788716f --- /dev/null +++ b/app/workers/move_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class MoveWorker + include Sidekiq::Worker + + def perform(source_account_id, target_account_id) + @source_account = Account.find(source_account_id) + @target_account = Account.find(target_account_id) + + if @target_account.local? + rewrite_follows! + else + queue_follow_unfollows! + end + rescue ActiveRecord::RecordNotFound + true + end + + private + + def rewrite_follows! + @source_account.passive_relationships + .where(account: Account.local) + .in_batches + .update_all(target_account: @target_account) + end + + def queue_follow_unfollows! + @source_account.followers.local.select(:id).find_in_batches do |accounts| + UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id] } + end + end +end -- cgit From 3babf8464b0903b854ec16d355909444ef3ca0bc Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 29 Sep 2019 22:58:01 +0200 Subject: Add voters count support (#11917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add voters count to polls * Add ActivityPub serialization and parsing of voters count * Add support for voters count in WebUI * Move incrementation of voters count out of redis lock * Reword “voters” to “people” --- app/javascript/mastodon/components/poll.js | 19 +++++++--- app/lib/activitypub/activity/create.rb | 40 +++++++++++++++++++--- app/lib/activitypub/adapter.rb | 1 + app/models/poll.rb | 1 + app/serializers/activitypub/note_serializer.rb | 12 ++++++- app/serializers/rest/poll_serializer.rb | 2 +- app/services/activitypub/process_poll_service.rb | 5 ++- app/services/post_status_service.rb | 2 +- app/services/vote_service.rb | 32 +++++++++++++++-- app/views/statuses/_poll.html.haml | 8 +++-- config/locales/en.yml | 3 ++ .../20190927232842_add_voters_count_to_polls.rb | 5 +++ db/schema.rb | 3 +- 13 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 db/migrate/20190927232842_add_voters_count_to_polls.rb (limited to 'app/lib/activitypub/activity') diff --git a/app/javascript/mastodon/components/poll.js b/app/javascript/mastodon/components/poll.js index f88d260f2..cdbcf8f70 100644 --- a/app/javascript/mastodon/components/poll.js +++ b/app/javascript/mastodon/components/poll.js @@ -102,10 +102,11 @@ class Poll extends ImmutablePureComponent { renderOption (option, optionIndex, showResults) { const { poll, disabled, intl } = this.props; - const percent = poll.get('votes_count') === 0 ? 0 : (option.get('votes_count') / poll.get('votes_count')) * 100; - const leading = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count')); - const active = !!this.state.selected[`${optionIndex}`]; - const voted = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex)); + const pollVotesCount = poll.get('voters_count') || poll.get('votes_count'); + const percent = pollVotesCount === 0 ? 0 : (option.get('votes_count') / pollVotesCount) * 100; + const leading = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count')); + const active = !!this.state.selected[`${optionIndex}`]; + const voted = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex)); let titleEmojified = option.get('title_emojified'); if (!titleEmojified) { @@ -157,6 +158,14 @@ class Poll extends ImmutablePureComponent { const showResults = poll.get('voted') || expired; const disabled = this.props.disabled || Object.entries(this.state.selected).every(item => !item); + let votesCount = null; + + if (poll.get('voters_count') !== null && poll.get('voters_count') !== undefined) { + votesCount = ; + } else { + votesCount = ; + } + return (
    @@ -166,7 +175,7 @@ class Poll extends ImmutablePureComponent {
    {!showResults && } {showResults && !this.props.disabled && · } - + {votesCount} {poll.get('expires_at') && · {timeRemaining}}
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index e69193b71..76bf9b2e5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -232,25 +232,40 @@ class ActivityPub::Activity::Create < ActivityPub::Activity items = @object['oneOf'] end + voters_count = @object['votersCount'] + @account.polls.new( multiple: multiple, expires_at: expires_at, options: items.map { |item| item['name'].presence || item['content'] }.compact, - cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 } + cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }, + voters_count: voters_count ) end def poll_vote? return false if replied_to_status.nil? || replied_to_status.preloadable_poll.nil? || !replied_to_status.local? || !replied_to_status.preloadable_poll.options.include?(@object['name']) - unless replied_to_status.preloadable_poll.expired? - replied_to_status.preloadable_poll.votes.create!(account: @account, choice: replied_to_status.preloadable_poll.options.index(@object['name']), uri: @object['id']) - ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals? - end + poll_vote! unless replied_to_status.preloadable_poll.expired? true end + def poll_vote! + poll = replied_to_status.preloadable_poll + already_voted = true + RedisLock.acquire(poll_lock_options) do |lock| + if lock.acquired? + already_voted = poll.votes.where(account: @account).exists? + poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: @object['id']) + else + raise Mastodon::RaceConditionError + end + end + increment_voters_count! unless already_voted + ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals? + end + def resolve_thread(status) return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri) ThreadResolveWorker.perform_async(status.id, in_reply_to_uri) @@ -416,7 +431,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) end + def increment_voters_count! + poll = replied_to_status.preloadable_poll + unless poll.voters_count.nil? + poll.voters_count = poll.voters_count + 1 + poll.save + end + rescue ActiveRecord::StaleObjectError + poll.reload + retry + end + def lock_options { redis: Redis.current, key: "create:#{@object['id']}" } end + + def poll_lock_options + { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" } + end end diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index cb2ac72d4..2a8f72333 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -21,6 +21,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' }, blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' }, discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' }, + voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' }, }.freeze def self.default_key_transform diff --git a/app/models/poll.rb b/app/models/poll.rb index 55a8f13a6..5427368fd 100644 --- a/app/models/poll.rb +++ b/app/models/poll.rb @@ -16,6 +16,7 @@ # created_at :datetime not null # updated_at :datetime not null # lock_version :integer default(0), not null +# voters_count :bigint(8) # class Poll < ApplicationRecord diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 364d3eda5..110621a28 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class ActivityPub::NoteSerializer < ActivityPub::Serializer - context_extensions :atom_uri, :conversation, :sensitive + context_extensions :atom_uri, :conversation, :sensitive, :voters_count attributes :id, :type, :summary, :in_reply_to, :published, :url, @@ -23,6 +23,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer attribute :end_time, if: :poll_and_expires? attribute :closed, if: :poll_and_expired? + attribute :voters_count, if: :poll_and_voters_count? + def id ActivityPub::TagManager.instance.uri_for(object) end @@ -141,6 +143,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer alias end_time closed + def voters_count + object.preloadable_poll.voters_count + end + def poll_and_expires? object.preloadable_poll&.expires_at&.present? end @@ -149,6 +155,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer object.preloadable_poll&.expired? end + def poll_and_voters_count? + object.preloadable_poll&.voters_count + end + class MediaAttachmentSerializer < ActivityPub::Serializer context_extensions :blurhash, :focal_point diff --git a/app/serializers/rest/poll_serializer.rb b/app/serializers/rest/poll_serializer.rb index eb98bb2d2..df6ebd0d4 100644 --- a/app/serializers/rest/poll_serializer.rb +++ b/app/serializers/rest/poll_serializer.rb @@ -2,7 +2,7 @@ class REST::PollSerializer < ActiveModel::Serializer attributes :id, :expires_at, :expired, - :multiple, :votes_count + :multiple, :votes_count, :voters_count has_many :loaded_options, key: :options has_many :emojis, serializer: REST::CustomEmojiSerializer diff --git a/app/services/activitypub/process_poll_service.rb b/app/services/activitypub/process_poll_service.rb index 2fbce65b9..cb4a0d460 100644 --- a/app/services/activitypub/process_poll_service.rb +++ b/app/services/activitypub/process_poll_service.rb @@ -28,6 +28,8 @@ class ActivityPub::ProcessPollService < BaseService end end + voters_count = @json['votersCount'] + latest_options = items.map { |item| item['name'].presence || item['content'] } # If for some reasons the options were changed, it invalidates all previous @@ -39,7 +41,8 @@ class ActivityPub::ProcessPollService < BaseService last_fetched_at: Time.now.utc, expires_at: expires_at, options: latest_options, - cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 } + cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }, + voters_count: voters_count ) rescue ActiveRecord::StaleObjectError poll.reload diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 34ec6d504..a0a650d62 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -174,7 +174,7 @@ class PostStatusService < BaseService def poll_attributes return if @options[:poll].blank? - @options[:poll].merge(account: @account) + @options[:poll].merge(account: @account, voters_count: 0) end def scheduled_options diff --git a/app/services/vote_service.rb b/app/services/vote_service.rb index 0eeb8fd56..cb7dce6e8 100644 --- a/app/services/vote_service.rb +++ b/app/services/vote_service.rb @@ -12,12 +12,24 @@ class VoteService < BaseService @choices = choices @votes = [] - ApplicationRecord.transaction do - @choices.each do |choice| - @votes << @poll.votes.create!(account: @account, choice: choice) + already_voted = true + + RedisLock.acquire(lock_options) do |lock| + if lock.acquired? + already_voted = @poll.votes.where(account: @account).exists? + + ApplicationRecord.transaction do + @choices.each do |choice| + @votes << @poll.votes.create!(account: @account, choice: choice) + end + end + else + raise Mastodon::RaceConditionError end end + increment_voters_count! unless already_voted + ActivityTracker.increment('activity:interactions') if @poll.account.local? @@ -53,4 +65,18 @@ class VoteService < BaseService def build_json(vote) Oj.dump(serialize_payload(vote, ActivityPub::VoteSerializer)) end + + def increment_voters_count! + unless @poll.voters_count.nil? + @poll.voters_count = @poll.voters_count + 1 + @poll.save + end + rescue ActiveRecord::StaleObjectError + @poll.reload + retry + end + + def lock_options + { redis: Redis.current, key: "vote:#{@poll.id}:#{@account.id}" } + end end diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml index d6b36a5d1..d1aba6ef9 100644 --- a/app/views/statuses/_poll.html.haml +++ b/app/views/statuses/_poll.html.haml @@ -1,12 +1,13 @@ - show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired? - own_votes = user_signed_in? ? poll.own_votes(current_account) : [] +- total_votes_count = poll.voters_count || poll.votes_count .poll %ul - poll.loaded_options.each_with_index do |option, index| %li - if show_results - - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0 + - percent = total_votes_count > 0 ? 100 * option.votes_count / total_votes_count : 0 %span.poll__chart{ style: "width: #{percent}%" } %label.poll__text>< @@ -24,7 +25,10 @@ %button.button.button-secondary{ disabled: true } = t('statuses.poll.vote') - %span= t('statuses.poll.total_votes', count: poll.votes_count) + - if poll.voters_count.nil? + %span= t('statuses.poll.total_votes', count: poll.votes_count) + - else + %span= t('statuses.poll.total_people', count: poll.voters_count) - unless poll.expires_at.nil? · diff --git a/config/locales/en.yml b/config/locales/en.yml index dbdfe0ca0..82e20cb1f 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1030,6 +1030,9 @@ en: private: Non-public toot cannot be pinned reblog: A boost cannot be pinned poll: + total_people: + one: "%{count} person" + other: "%{count} people" total_votes: one: "%{count} vote" other: "%{count} votes" diff --git a/db/migrate/20190927232842_add_voters_count_to_polls.rb b/db/migrate/20190927232842_add_voters_count_to_polls.rb new file mode 100644 index 000000000..846385700 --- /dev/null +++ b/db/migrate/20190927232842_add_voters_count_to_polls.rb @@ -0,0 +1,5 @@ +class AddVotersCountToPolls < ActiveRecord::Migration[5.2] + def change + add_column :polls, :voters_count, :bigint + end +end diff --git a/db/schema.rb b/db/schema.rb index 8eeaf48a0..557b777e0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_09_27_124642) do +ActiveRecord::Schema.define(version: 2019_09_27_232842) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -529,6 +529,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_124642) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "lock_version", default: 0, null: false + t.bigint "voters_count" t.index ["account_id"], name: "index_polls_on_account_id" t.index ["status_id"], name: "index_polls_on_status_id" end -- cgit From d2919f7e94a22971fb368f614ecccce2b055d231 Mon Sep 17 00:00:00 2001 From: puckipedia Date: Thu, 24 Oct 2019 20:45:43 +0000 Subject: Allow Accept/Reject with a non-embedded object (#12199) Some ActivityPub servers refuse to embed remote objects into their own output. This is because they are not the authoritative source for these objects, and as such embedding them is always a waste of space. The follow request and follow models contain a URI, so this can be used to match them. --- app/lib/activitypub/activity.rb | 8 ++++++++ app/lib/activitypub/activity/accept.rb | 9 +++++---- app/lib/activitypub/activity/reject.rb | 10 ++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index a4a9baaee..5cd48a6ae 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -153,6 +153,14 @@ class ActivityPub::Activity fetch_remote_original_status end + def follow_request_from_object + @follow_request ||= FollowRequest.find_by(target_account: @account, uri: object_uri) unless object_uri.nil? + end + + def follow_from_object + @follow ||= Follow.find_by(target_account: @account, uri: object_uri) unless object_uri.nil? + end + def fetch_remote_original_status if object_uri.start_with?('http') return if ActivityPub::TagManager.instance.local_uri?(object_uri) diff --git a/app/lib/activitypub/activity/accept.rb b/app/lib/activitypub/activity/accept.rb index 348ee0d1c..7010ff43e 100644 --- a/app/lib/activitypub/activity/accept.rb +++ b/app/lib/activitypub/activity/accept.rb @@ -2,17 +2,18 @@ class ActivityPub::Activity::Accept < ActivityPub::Activity def perform + return accept_follow_for_relay if relay_follow? + return follow_request_from_object.authorize! unless follow_request_from_object.nil? + case @object['type'] when 'Follow' - accept_follow + accept_embedded_follow end end private - def accept_follow - return accept_follow_for_relay if relay_follow? - + def accept_embedded_follow target_account = account_from_uri(target_uri) return if target_account.nil? || !target_account.local? diff --git a/app/lib/activitypub/activity/reject.rb b/app/lib/activitypub/activity/reject.rb index dba21fb9a..8d771ed81 100644 --- a/app/lib/activitypub/activity/reject.rb +++ b/app/lib/activitypub/activity/reject.rb @@ -2,17 +2,19 @@ class ActivityPub::Activity::Reject < ActivityPub::Activity def perform + return reject_follow_for_relay if relay_follow? + return follow_request_from_object.reject! unless follow_request_from_object.nil? + return UnfollowService.new.call(follow_from_object.target_account, @account) unless follow_from_object.nil? + case @object['type'] when 'Follow' - reject_follow + reject_embedded_follow end end private - def reject_follow - return reject_follow_for_relay if relay_follow? - + def reject_embedded_follow target_account = account_from_uri(target_uri) return if target_account.nil? || !target_account.local? -- cgit From f05b0463dbc83a85dee7490e07bd79859132ee48 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 30 Nov 2019 19:58:00 +0100 Subject: Fallback to Create audience when object has no defined audience (#12249) Fixes #11137 --- app/lib/activitypub/activity/create.rb | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'app/lib/activitypub/activity') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 76bf9b2e5..8a12a2b08 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -25,6 +25,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity private + def audience_to + @object['to'] || @json['to'] + end + + def audience_cc + @object['cc'] || @json['cc'] + end + def process_status @tags = [] @mentions = [] @@ -75,7 +83,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def process_audience - (as_array(@object['to']) + as_array(@object['cc'])).uniq.each do |audience| + (as_array(audience_to) + as_array(audience_cc)).uniq.each do |audience| next if audience == ActivityPub::TagManager::COLLECTIONS[:public] # Unlike with tags, there is no point in resolving accounts we don't already @@ -291,11 +299,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def visibility_from_audience - if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public]) + if equals_or_includes?(audience_to, ActivityPub::TagManager::COLLECTIONS[:public]) :public - elsif equals_or_includes?(@object['cc'], ActivityPub::TagManager::COLLECTIONS[:public]) + elsif equals_or_includes?(audience_cc, ActivityPub::TagManager::COLLECTIONS[:public]) :unlisted - elsif equals_or_includes?(@object['to'], @account.followers_url) + elsif equals_or_includes?(audience_to, @account.followers_url) :private else :direct @@ -304,7 +312,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def audience_includes?(account) uri = ActivityPub::TagManager.instance.uri_for(account) - equals_or_includes?(@object['to'], uri) || equals_or_includes?(@object['cc'], uri) + equals_or_includes?(audience_to, uri) || equals_or_includes?(audience_cc, uri) end def replied_to_status @@ -415,7 +423,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def addresses_local_accounts? return true if @options[:delivered_to_account_id] - local_usernames = (as_array(@object['to']) + as_array(@object['cc'])).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } + local_usernames = (as_array(audience_to) + as_array(audience_cc)).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } return false if local_usernames.empty? -- cgit