From b8514561394767a10d3cf40132ada24d938c1680 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 7 Jul 2019 16:16:51 +0200 Subject: Remove Atom feeds and old URLs in the form of `GET /:username/updates/:id` (#11247) --- app/helpers/admin/action_logs_helper.rb | 2 +- app/helpers/home_helper.rb | 2 +- app/helpers/statuses_helper.rb | 222 ++++++++++++++++++++++++++++++++ app/helpers/stream_entries_helper.rb | 220 ------------------------------- 4 files changed, 224 insertions(+), 222 deletions(-) create mode 100644 app/helpers/statuses_helper.rb delete mode 100644 app/helpers/stream_entries_helper.rb (limited to 'app/helpers') diff --git a/app/helpers/admin/action_logs_helper.rb b/app/helpers/admin/action_logs_helper.rb index e5fbb1500..1daa60774 100644 --- a/app/helpers/admin/action_logs_helper.rb +++ b/app/helpers/admin/action_logs_helper.rb @@ -89,7 +89,7 @@ module Admin::ActionLogsHelper when 'DomainBlock', 'EmailDomainBlock' link_to record.domain, "https://#{record.domain}" when 'Status' - link_to record.account.acct, TagManager.instance.url_for(record) + link_to record.account.acct, ActivityPub::TagManager.instance.url_for(record) when 'AccountWarning' link_to record.target_account.acct, admin_account_path(record.target_account_id) end diff --git a/app/helpers/home_helper.rb b/app/helpers/home_helper.rb index df60b7dd7..b66e827fe 100644 --- a/app/helpers/home_helper.rb +++ b/app/helpers/home_helper.rb @@ -21,7 +21,7 @@ module HomeHelper end end else - link_to(path || TagManager.instance.url_for(account), class: 'account__display-name') do + link_to(path || ActivityPub::TagManager.instance.url_for(account), class: 'account__display-name') do content_tag(:div, class: 'account__avatar-wrapper') do content_tag(:div, '', class: 'account__avatar', style: "width: #{size}px; height: #{size}px; background-size: #{size}px #{size}px; background-image: url(#{full_asset_url(current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url)})") end + diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb new file mode 100644 index 000000000..e067380f6 --- /dev/null +++ b/app/helpers/statuses_helper.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +module StatusesHelper + EMBEDDED_CONTROLLER = 'statuses' + EMBEDDED_ACTION = 'embed' + + def display_name(account, **options) + if options[:custom_emojify] + Formatter.instance.format_display_name(account, options) + else + account.display_name.presence || account.username + end + end + + def account_action_button(account) + if user_signed_in? + if account.id == current_user.account_id + link_to settings_profile_url, class: 'button logo-button' do + safe_join([svg_logo, t('settings.edit_profile')]) + end + elsif current_account.following?(account) || current_account.requested?(account) + link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do + safe_join([svg_logo, t('accounts.unfollow')]) + end + elsif !(account.memorial? || account.moved?) + link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do + safe_join([svg_logo, t('accounts.follow')]) + end + end + elsif !(account.memorial? || account.moved?) + link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do + safe_join([svg_logo, t('accounts.follow')]) + end + end + end + + def svg_logo + content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976') + end + + def svg_logo_full + content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678') + end + + def account_badge(account, all: false) + if account.bot? + content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles') + elsif (Setting.show_staff_badge && account.user_staff?) || all + content_tag(:div, class: 'roles') do + if all && !account.user_staff? + content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role') + elsif account.user_admin? + content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin') + elsif account.user_moderator? + content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator') + end + end + end + end + + def link_to_more(url) + link_to t('statuses.show_more'), url, class: 'load-more load-gap' + end + + def nothing_here(extra_classes = '') + content_tag(:div, class: "nothing-here #{extra_classes}") do + t('accounts.nothing_here') + end + end + + def account_description(account) + prepend_str = [ + [ + number_to_human(account.statuses_count, strip_insignificant_zeros: true), + I18n.t('accounts.posts', count: account.statuses_count), + ].join(' '), + + [ + number_to_human(account.following_count, strip_insignificant_zeros: true), + I18n.t('accounts.following', count: account.following_count), + ].join(' '), + + [ + number_to_human(account.followers_count, strip_insignificant_zeros: true), + I18n.t('accounts.followers', count: account.followers_count), + ].join(' '), + ].join(', ') + + [prepend_str, account.note].join(' · ') + end + + def media_summary(status) + attachments = { image: 0, video: 0 } + + status.media_attachments.each do |media| + if media.video? + attachments[:video] += 1 + else + attachments[:image] += 1 + end + end + + text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ') + + return if text.blank? + + I18n.t('statuses.attached.description', attached: text) + end + + def status_text_summary(status) + return if status.spoiler_text.blank? + + I18n.t('statuses.content_warning', warning: status.spoiler_text) + end + + def poll_summary(status) + return unless status.preloadable_poll + + status.preloadable_poll.options.map { |o| "[ ] #{o}" }.join("\n") + end + + def status_description(status) + components = [[media_summary(status), status_text_summary(status)].reject(&:blank?).join(' · ')] + + if status.spoiler_text.blank? + components << status.text + components << poll_summary(status) + end + + components.reject(&:blank?).join("\n\n") + end + + def stream_link_target + embedded_view? ? '_blank' : nil + end + + def acct(account) + if account.local? + "@#{account.acct}@#{Rails.configuration.x.local_domain}" + else + "@#{account.acct}" + end + end + + def style_classes(status, is_predecessor, is_successor, include_threads) + classes = ['entry'] + classes << 'entry-predecessor' if is_predecessor + classes << 'entry-reblog' if status.reblog? + classes << 'entry-successor' if is_successor + classes << 'entry-center' if include_threads + classes.join(' ') + end + + def microformats_classes(status, is_direct_parent, is_direct_child) + classes = [] + classes << 'p-in-reply-to' if is_direct_parent + classes << 'p-repost-of' if status.reblog? && is_direct_parent + classes << 'p-comment' if is_direct_child + classes.join(' ') + end + + def microformats_h_class(status, is_predecessor, is_successor, include_threads) + if is_predecessor || status.reblog? || is_successor + 'h-cite' + elsif include_threads + '' + else + 'h-entry' + end + end + + def rtl_status?(status) + status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text)) + end + + def rtl?(text) + text = simplified_text(text) + rtl_words = text.scan(/[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}]+/m) + + if rtl_words.present? + total_size = text.size.to_f + rtl_size(rtl_words) / total_size > 0.3 + else + false + end + end + + def fa_visibility_icon(status) + case status.visibility + when 'public' + fa_icon 'globe fw' + when 'unlisted' + fa_icon 'unlock fw' + when 'private' + fa_icon 'lock fw' + when 'direct' + fa_icon 'envelope fw' + end + end + + private + + def simplified_text(text) + text.dup.tap do |new_text| + URI.extract(new_text).each do |url| + new_text.gsub!(url, '') + end + + new_text.gsub!(Account::MENTION_RE, '') + new_text.gsub!(Tag::HASHTAG_RE, '') + new_text.gsub!(/\s+/, '') + end + end + + def rtl_size(words) + words.reduce(0) { |acc, elem| acc + elem.size }.to_f + end + + def embedded_view? + params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION + end +end diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb deleted file mode 100644 index 02a860a74..000000000 --- a/app/helpers/stream_entries_helper.rb +++ /dev/null @@ -1,220 +0,0 @@ -# frozen_string_literal: true - -module StreamEntriesHelper - EMBEDDED_CONTROLLER = 'statuses' - EMBEDDED_ACTION = 'embed' - - def display_name(account, **options) - if options[:custom_emojify] - Formatter.instance.format_display_name(account, options) - else - account.display_name.presence || account.username - end - end - - def account_action_button(account) - if user_signed_in? - if account.id == current_user.account_id - link_to settings_profile_url, class: 'button logo-button' do - safe_join([svg_logo, t('settings.edit_profile')]) - end - elsif current_account.following?(account) || current_account.requested?(account) - link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do - safe_join([svg_logo, t('accounts.unfollow')]) - end - elsif !(account.memorial? || account.moved?) - link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do - safe_join([svg_logo, t('accounts.follow')]) - end - end - elsif !(account.memorial? || account.moved?) - link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do - safe_join([svg_logo, t('accounts.follow')]) - end - end - end - - def svg_logo - content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976') - end - - def svg_logo_full - content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678') - end - - def account_badge(account, all: false) - if account.bot? - content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles') - elsif (Setting.show_staff_badge && account.user_staff?) || all - content_tag(:div, class: 'roles') do - if all && !account.user_staff? - content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role') - elsif account.user_admin? - content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin') - elsif account.user_moderator? - content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator') - end - end - end - end - - def link_to_more(url) - link_to t('statuses.show_more'), url, class: 'load-more load-gap' - end - - def nothing_here(extra_classes = '') - content_tag(:div, class: "nothing-here #{extra_classes}") do - t('accounts.nothing_here') - end - end - - def account_description(account) - prepend_str = [ - [ - number_to_human(account.statuses_count, strip_insignificant_zeros: true), - I18n.t('accounts.posts', count: account.statuses_count), - ].join(' '), - - [ - number_to_human(account.following_count, strip_insignificant_zeros: true), - I18n.t('accounts.following', count: account.following_count), - ].join(' '), - - [ - number_to_human(account.followers_count, strip_insignificant_zeros: true), - I18n.t('accounts.followers', count: account.followers_count), - ].join(' '), - ].join(', ') - - [prepend_str, account.note].join(' · ') - end - - def media_summary(status) - attachments = { image: 0, video: 0 } - - status.media_attachments.each do |media| - if media.video? - attachments[:video] += 1 - else - attachments[:image] += 1 - end - end - - text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ') - - return if text.blank? - - I18n.t('statuses.attached.description', attached: text) - end - - def status_text_summary(status) - return if status.spoiler_text.blank? - I18n.t('statuses.content_warning', warning: status.spoiler_text) - end - - def poll_summary(status) - return unless status.preloadable_poll - status.preloadable_poll.options.map { |o| "[ ] #{o}" }.join("\n") - end - - def status_description(status) - components = [[media_summary(status), status_text_summary(status)].reject(&:blank?).join(' · ')] - - if status.spoiler_text.blank? - components << status.text - components << poll_summary(status) - end - - components.reject(&:blank?).join("\n\n") - end - - def stream_link_target - embedded_view? ? '_blank' : nil - end - - def acct(account) - if account.local? - "@#{account.acct}@#{Rails.configuration.x.local_domain}" - else - "@#{account.acct}" - end - end - - def style_classes(status, is_predecessor, is_successor, include_threads) - classes = ['entry'] - classes << 'entry-predecessor' if is_predecessor - classes << 'entry-reblog' if status.reblog? - classes << 'entry-successor' if is_successor - classes << 'entry-center' if include_threads - classes.join(' ') - end - - def microformats_classes(status, is_direct_parent, is_direct_child) - classes = [] - classes << 'p-in-reply-to' if is_direct_parent - classes << 'p-repost-of' if status.reblog? && is_direct_parent - classes << 'p-comment' if is_direct_child - classes.join(' ') - end - - def microformats_h_class(status, is_predecessor, is_successor, include_threads) - if is_predecessor || status.reblog? || is_successor - 'h-cite' - elsif include_threads - '' - else - 'h-entry' - end - end - - def rtl_status?(status) - status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text)) - end - - def rtl?(text) - text = simplified_text(text) - rtl_words = text.scan(/[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}]+/m) - - if rtl_words.present? - total_size = text.size.to_f - rtl_size(rtl_words) / total_size > 0.3 - else - false - end - end - - def fa_visibility_icon(status) - case status.visibility - when 'public' - fa_icon 'globe fw' - when 'unlisted' - fa_icon 'unlock fw' - when 'private' - fa_icon 'lock fw' - when 'direct' - fa_icon 'envelope fw' - end - end - - private - - def simplified_text(text) - text.dup.tap do |new_text| - URI.extract(new_text).each do |url| - new_text.gsub!(url, '') - end - - new_text.gsub!(Account::MENTION_RE, '') - new_text.gsub!(Tag::HASHTAG_RE, '') - new_text.gsub!(/\s+/, '') - end - end - - def rtl_size(words) - words.reduce(0) { |acc, elem| acc + elem.size }.to_f - end - - def embedded_view? - params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION - end -end -- cgit From 4e921832272425352d28cad550bfc4dffd6d0e78 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 9 Jul 2019 03:27:35 +0200 Subject: Refactor domain block checks (#11268) --- app/controllers/concerns/signature_verification.rb | 4 + app/helpers/domain_control_helper.rb | 17 ++++ app/lib/tag_manager.rb | 3 + .../fetch_featured_collection_service.rb | 3 +- .../activitypub/fetch_remote_account_service.rb | 14 ++- .../activitypub/fetch_remote_poll_service.rb | 2 + .../activitypub/process_account_service.rb | 5 +- .../activitypub/process_collection_service.rb | 4 +- app/services/activitypub/process_poll_service.rb | 1 + app/services/resolve_account_service.rb | 101 +++++++++++++-------- spec/services/resolve_account_service_spec.rb | 5 + 11 files changed, 108 insertions(+), 51 deletions(-) create mode 100644 app/helpers/domain_control_helper.rb (limited to 'app/helpers') diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb index 90a57197c..0ccdf5ec9 100644 --- a/app/controllers/concerns/signature_verification.rb +++ b/app/controllers/concerns/signature_verification.rb @@ -5,6 +5,8 @@ module SignatureVerification extend ActiveSupport::Concern + include DomainControlHelper + def signed_request? request.headers['Signature'].present? end @@ -126,6 +128,8 @@ module SignatureVerification if key_id.start_with?('acct:') stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, '')) } elsif !ActivityPub::TagManager.instance.local_uri?(key_id) + return if domain_not_allowed?(key_id) + account = ActivityPub::TagManager.instance.uri_to_resource(key_id, Account) account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false) } account diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb new file mode 100644 index 000000000..efd328f81 --- /dev/null +++ b/app/helpers/domain_control_helper.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module DomainControlHelper + def domain_not_allowed?(uri_or_domain) + return if uri_or_domain.blank? + + domain = begin + if uri_or_domain.include?('://') + Addressable::URI.parse(uri_or_domain).domain + else + uri_or_domain + end + end + + DomainBlock.blocked?(domain) + end +end diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index daf4f556b..c88cf4994 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -24,13 +24,16 @@ class TagManager def same_acct?(canonical, needle) return true if canonical.casecmp(needle).zero? + username, domain = needle.split('@') + local_domain?(domain) && canonical.casecmp(username).zero? end def local_url?(url) uri = Addressable::URI.parse(url).normalize domain = uri.host + (uri.port ? ":#{uri.port}" : '') + TagManager.instance.web_domain?(domain) end end diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb index 6a137b520..2c2770466 100644 --- a/app/services/activitypub/fetch_featured_collection_service.rb +++ b/app/services/activitypub/fetch_featured_collection_service.rb @@ -4,13 +4,12 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService include JsonLdHelper def call(account) - return if account.featured_collection_url.blank? + return if account.featured_collection_url.blank? || account.suspended? || account.local? @account = account @json = fetch_resource(@account.featured_collection_url, true) return unless supported_context? - return if @account.suspended? || @account.local? case @json['type'] when 'Collection', 'CollectionPage' diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 3c2044941..d65c8f951 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -2,18 +2,22 @@ class ActivityPub::FetchRemoteAccountService < BaseService include JsonLdHelper + include DomainControlHelper SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze # Does a WebFinger roundtrip on each call, unless `only_key` is true def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false) + return if domain_not_allowed?(uri) return ActivityPub::TagManager.instance.uri_to_resource(uri, Account) if ActivityPub::TagManager.instance.local_uri?(uri) - @json = if prefetched_body.nil? - fetch_resource(uri, id) - else - body_to_json(prefetched_body, compare_id: id ? uri : nil) - end + @json = begin + if prefetched_body.nil? + fetch_resource(uri, id) + else + body_to_json(prefetched_body, compare_id: id ? uri : nil) + end + end return if !supported_context? || !expected_type? || (break_on_redirect && @json['movedTo'].present?) diff --git a/app/services/activitypub/fetch_remote_poll_service.rb b/app/services/activitypub/fetch_remote_poll_service.rb index 854a32d05..1c79ecf11 100644 --- a/app/services/activitypub/fetch_remote_poll_service.rb +++ b/app/services/activitypub/fetch_remote_poll_service.rb @@ -5,7 +5,9 @@ class ActivityPub::FetchRemotePollService < BaseService def call(poll, on_behalf_of = nil) json = fetch_resource(poll.status.uri, true, on_behalf_of) + return unless supported_context?(json) + ActivityPub::ProcessPollService.new.call(poll, json) end end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 3857e7c16..603e27ed9 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -2,11 +2,12 @@ class ActivityPub::ProcessAccountService < BaseService include JsonLdHelper + include DomainControlHelper # Should be called with confirmed valid JSON # and WebFinger-resolved username and domain def call(username, domain, json, options = {}) - return if json['inbox'].blank? || unsupported_uri_scheme?(json['id']) + return if json['inbox'].blank? || unsupported_uri_scheme?(json['id']) || domain_not_allowed?(domain) @options = options @json = json @@ -15,8 +16,6 @@ class ActivityPub::ProcessAccountService < BaseService @domain = domain @collections = {} - return if auto_suspend? - RedisLock.acquire(lock_options) do |lock| if lock.acquired? @account = Account.find_remote(@username, @domain) diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb index 881df478b..a2a2e7071 100644 --- a/app/services/activitypub/process_collection_service.rb +++ b/app/services/activitypub/process_collection_service.rb @@ -8,9 +8,7 @@ class ActivityPub::ProcessCollectionService < BaseService @json = Oj.load(body, mode: :strict) @options = options - return unless supported_context? - return if different_actor? && verify_account!.nil? - return if @account.suspended? || @account.local? + return if !supported_context? || (different_actor? && verify_account!.nil?) || @account.suspended? || @account.local? case @json['type'] when 'Collection', 'CollectionPage' diff --git a/app/services/activitypub/process_poll_service.rb b/app/services/activitypub/process_poll_service.rb index 61357abd3..2fbce65b9 100644 --- a/app/services/activitypub/process_poll_service.rb +++ b/app/services/activitypub/process_poll_service.rb @@ -5,6 +5,7 @@ class ActivityPub::ProcessPollService < BaseService def call(poll, json) @json = json + return unless expected_type? previous_expires_at = poll.expires_at diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 0ea31a0d8..41a2eb158 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -1,75 +1,108 @@ # frozen_string_literal: true -require_relative '../models/account' - class ResolveAccountService < BaseService include JsonLdHelper + include DomainControlHelper + + class WebfingerRedirectError < StandardError; end - # Find or create a local account for a remote user. - # When creating, look up the user's webfinger and fetch all - # important information from their feed - # @param [String, Account] uri User URI in the form of username@domain + # Find or create an account record for a remote user. When creating, + # look up the user's webfinger and fetch ActivityPub data + # @param [String, Account] uri URI in the username@domain format or account record # @param [Hash] options + # @option options [Boolean] :redirected Do not follow further Webfinger redirects + # @option options [Boolean] :skip_webfinger Do not attempt to refresh account data # @return [Account] def call(uri, options = {}) + return if uri.blank? + + process_options!(uri, options) + + # First of all we want to check if we've got the account + # record with the URI already, and if so, we can exit early + + return if domain_not_allowed?(@domain) + + @account ||= Account.find_remote(@username, @domain) + + return @account if @account&.local? || !webfinger_update_due? + + # At this point we are in need of a Webfinger query, which may + # yield us a different username/domain through a redirect + + process_webfinger! + + # Because the username/domain pair may be different than what + # we already checked, we need to check if we've already got + # the record with that URI, again + + return if domain_not_allowed?(@domain) + + @account ||= Account.find_remote(@username, @domain) + + return @account if @account&.local? || !webfinger_update_due? + + # Now it is certain, it is definitely a remote account, and it + # either needs to be created, or updated from fresh data + + process_account! + rescue Goldfinger::Error, WebfingerRedirectError, Oj::ParseError => e + Rails.logger.debug "Webfinger query for #{@uri} failed: #{e}" + nil + end + + private + + def process_options!(uri, options) @options = options if uri.is_a?(Account) @account = uri @username = @account.username @domain = @account.domain - uri = "#{@username}@#{@domain}" - - return @account if @account.local? || !webfinger_update_due? + @uri = [@username, @domain].compact.join('@') else + @uri = uri @username, @domain = uri.split('@') - - return Account.find_local(@username) if TagManager.instance.local_domain?(@domain) - - @account = Account.find_remote(@username, @domain) - - return @account unless webfinger_update_due? end - Rails.logger.debug "Looking up webfinger for #{uri}" - - @webfinger = Goldfinger.finger("acct:#{uri}") + @domain = nil if TagManager.instance.local_domain?(@domain) + end + def process_webfinger! + @webfinger = Goldfinger.finger("acct:#{@uri}") confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@') if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? @username = confirmed_username @domain = confirmed_domain - elsif options[:redirected].nil? - return call("#{confirmed_username}@#{confirmed_domain}", options.merge(redirected: true)) + elsif @options[:redirected].nil? + @account = ResolveAccountService.new.call("#{confirmed_username}@#{confirmed_domain}", @options.merge(redirected: true)) else - Rails.logger.debug 'Requested and returned acct URIs do not match' - return + raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" end - return Account.find_local(@username) if TagManager.instance.local_domain?(@domain) + @domain = nil if TagManager.instance.local_domain?(@domain) + end + + def process_account! return unless activitypub_ready? RedisLock.acquire(lock_options) do |lock| if lock.acquired? @account = Account.find_remote(@username, @domain) - next unless @account.nil? || @account.activitypub? + next if (@account.present? && !@account.activitypub?) || actor_json.nil? - handle_activitypub + @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json) else raise Mastodon::RaceConditionError end end @account - rescue Goldfinger::Error => e - Rails.logger.debug "Webfinger query for #{uri} unsuccessful: #{e}" - nil end - private - def webfinger_update_due? @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?) end @@ -78,14 +111,6 @@ class ResolveAccountService < BaseService !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) end - def handle_activitypub - return if actor_json.nil? - - @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json) - rescue Oj::ParseError - nil - end - def actor_url @actor_url ||= @webfinger.link('self').href end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 7a64f4161..cea942e39 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -53,6 +53,11 @@ RSpec.describe ResolveAccountService, type: :service do fail_occurred = false return_values = Concurrent::Array.new + # Preload classes that throw circular dependency errors in threads + Account + TagManager + DomainBlock + threads = Array.new(5) do Thread.new do true while wait_for_start -- cgit From 5d3feed191bcbe2769512119752b426108152fe9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 10 Jul 2019 18:59:28 +0200 Subject: Refactor fetching of remote resources (#11251) --- app/helpers/jsonld_helper.rb | 47 ++++++----- app/lib/request.rb | 2 +- .../activitypub/fetch_remote_status_service.rb | 20 ++--- app/services/fetch_atom_service.rb | 93 --------------------- app/services/fetch_link_card_service.rb | 2 +- app/services/fetch_remote_account_service.rb | 2 +- app/services/fetch_remote_status_service.rb | 2 +- app/services/fetch_resource_service.rb | 68 +++++++++++++++ app/services/resolve_url_service.rb | 47 ++++------- app/workers/activitypub/delivery_worker.rb | 16 ++-- spec/services/fetch_atom_service_spec.rb | 96 ---------------------- spec/services/fetch_remote_account_service_spec.rb | 1 + spec/services/fetch_resource_service_spec.rb | 96 ++++++++++++++++++++++ spec/services/resolve_url_service_spec.rb | 44 ++-------- 14 files changed, 231 insertions(+), 305 deletions(-) delete mode 100644 app/services/fetch_atom_service.rb create mode 100644 app/services/fetch_resource_service.rb delete mode 100644 spec/services/fetch_atom_service_spec.rb create mode 100644 spec/services/fetch_resource_service_spec.rb (limited to 'app/helpers') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 5b4011275..34a657e06 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -16,13 +16,15 @@ module JsonLdHelper # The url attribute can be a string, an array of strings, or an array of objects. # The objects could include a mimeType. Not-included mimeType means it's text/html. def url_to_href(value, preferred_type = nil) - single_value = if value.is_a?(Array) && !value.first.is_a?(String) - value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) } - elsif value.is_a?(Array) - value.first - else - value - end + single_value = begin + if value.is_a?(Array) && !value.first.is_a?(String) + value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) } + elsif value.is_a?(Array) + value.first + else + value + end + end if single_value.nil? || single_value.is_a?(String) single_value @@ -64,7 +66,9 @@ module JsonLdHelper def fetch_resource(uri, id, on_behalf_of = nil) unless id json = fetch_resource_without_id_validation(uri, on_behalf_of) + return unless json + uri = json['id'] end @@ -74,24 +78,26 @@ module JsonLdHelper def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false) build_request(uri, on_behalf_of).perform do |response| - unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - raise Mastodon::UnexpectedResponseError, response - end + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error + return body_to_json(response.body_with_limit) if response.code == 200 end + # If request failed, retry without doing it on behalf of a user return if on_behalf_of.nil? + build_request(uri).perform do |response| - unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - raise Mastodon::UnexpectedResponseError, response - end + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error + response.code == 200 ? body_to_json(response.body_with_limit) : nil end end def body_to_json(body, compare_id: nil) json = body.is_a?(String) ? Oj.load(body, mode: :strict) : body + return if compare_id.present? && json['id'] != compare_id + json rescue Oj::ParseError nil @@ -105,35 +111,34 @@ module JsonLdHelper end end - private - def response_successful?(response) (200...300).cover?(response.code) end def response_error_unsalvageable?(response) - (400...500).cover?(response.code) && response.code != 429 + response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) end def build_request(uri, on_behalf_of = nil) - request = Request.new(:get, uri) - request.on_behalf_of(on_behalf_of) if on_behalf_of - request.add_headers('Accept' => 'application/activity+json, application/ld+json') - request + Request.new(:get, uri).tap do |request| + request.on_behalf_of(on_behalf_of) if on_behalf_of + request.add_headers('Accept' => 'application/activity+json, application/ld+json') + end end def load_jsonld_context(url, _options = {}, &_block) json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do request = Request.new(:get, url) request.add_headers('Accept' => 'application/ld+json') - request.perform do |res| raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json' + res.body_with_limit end end doc = JSON::LD::API::RemoteDocument.new(url, json) + block_given? ? yield(doc) : doc end end diff --git a/app/lib/request.rb b/app/lib/request.rb index 322457ad7..1fd3f5190 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -41,7 +41,7 @@ class Request end def on_behalf_of(account, key_id_format = :acct, sign_with: nil) - raise ArgumentError unless account.local? + raise ArgumentError, 'account must be local' unless account&.local? @account = account @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 469821032..cf4f62899 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -5,18 +5,18 @@ class ActivityPub::FetchRemoteStatusService < BaseService # Should be called when uri has already been checked for locality def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil) - @json = if prefetched_body.nil? - fetch_resource(uri, id, on_behalf_of) - else - body_to_json(prefetched_body, compare_id: id ? uri : nil) - end + @json = begin + if prefetched_body.nil? + fetch_resource(uri, id, on_behalf_of) + else + body_to_json(prefetched_body, compare_id: id ? uri : nil) + end + end - return unless supported_context? && expected_type? - - return if actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id) + return if !(supported_context? && expected_type?) || actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id) actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account) - actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update(actor) + actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update?(actor) return if actor.nil? || actor.suspended? @@ -46,7 +46,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end - def needs_update(actor) + def needs_update?(actor) actor.possibly_stale? end end diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb deleted file mode 100644 index d6508a988..000000000 --- a/app/services/fetch_atom_service.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -class FetchAtomService < BaseService - include JsonLdHelper - - def call(url) - return if url.blank? - - result = process(url) - - # retry without ActivityPub - result ||= process(url) if @unsupported_activity - - result - rescue OpenSSL::SSL::SSLError => e - Rails.logger.debug "SSL error: #{e}" - nil - rescue HTTP::ConnectionError => e - Rails.logger.debug "HTTP ConnectionError: #{e}" - nil - end - - private - - def process(url, terminal = false) - @url = url - perform_request { |response| process_response(response, terminal) } - end - - def perform_request(&block) - accept = 'text/html' - accept = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/atom+xml, ' + accept unless @unsupported_activity - - Request.new(:get, @url).add_headers('Accept' => accept).perform(&block) - end - - def process_response(response, terminal = false) - return nil if response.code != 200 - - if response.mime_type == 'application/atom+xml' - [@url, { prefetched_body: response.body_with_limit }, :ostatus] - elsif ['application/activity+json', 'application/ld+json'].include?(response.mime_type) - body = response.body_with_limit - json = body_to_json(body) - if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present? - [json['id'], { prefetched_body: body, id: true }, :activitypub] - elsif supported_context?(json) && expected_type?(json) - [json['id'], { prefetched_body: body, id: true }, :activitypub] - else - @unsupported_activity = true - nil - end - elsif !terminal - link_header = response['Link'] && parse_link_header(response) - - if link_header&.find_link(%w(rel alternate)) - process_link_headers(link_header) - elsif response.mime_type == 'text/html' - process_html(response) - end - end - end - - def expected_type?(json) - equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) - end - - def process_html(response) - page = Nokogiri::HTML(response.body_with_limit) - - json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } - atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } - - result ||= process(json_link['href'], terminal: true) unless json_link.nil? || @unsupported_activity - result ||= process(atom_link['href'], terminal: true) unless atom_link.nil? - - result - end - - def process_link_headers(link_header) - json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) - atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml)) - - result ||= process(json_link.href, terminal: true) unless json_link.nil? || @unsupported_activity - result ||= process(atom_link.href, terminal: true) unless atom_link.nil? - - result - end - - def parse_link_header(response) - LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link']) - end -end diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 75fbd0e8c..4e75c370f 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -29,7 +29,7 @@ class FetchLinkCardService < BaseService end attach_card if @card&.persisted? - rescue HTTP::Error, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e Rails.logger.debug "Error fetching link #{@url}: #{e}" nil end diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb index a7f95603d..3cd06e30f 100644 --- a/app/services/fetch_remote_account_service.rb +++ b/app/services/fetch_remote_account_service.rb @@ -3,7 +3,7 @@ class FetchRemoteAccountService < BaseService def call(url, prefetched_body = nil, protocol = :ostatus) if prefetched_body.nil? - resource_url, resource_options, protocol = FetchAtomService.new.call(url) + resource_url, resource_options, protocol = FetchResourceService.new.call(url) else resource_url = url resource_options = { prefetched_body: prefetched_body } diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb index aac39dfd5..208dc7809 100644 --- a/app/services/fetch_remote_status_service.rb +++ b/app/services/fetch_remote_status_service.rb @@ -3,7 +3,7 @@ class FetchRemoteStatusService < BaseService def call(url, prefetched_body = nil, protocol = :ostatus) if prefetched_body.nil? - resource_url, resource_options, protocol = FetchAtomService.new.call(url) + resource_url, resource_options, protocol = FetchResourceService.new.call(url) else resource_url = url resource_options = { prefetched_body: prefetched_body } diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb new file mode 100644 index 000000000..c0473f3ad --- /dev/null +++ b/app/services/fetch_resource_service.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +class FetchResourceService < BaseService + include JsonLdHelper + + ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html' + + def call(url) + return if url.blank? + + process(url) + rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + Rails.logger.debug "Error fetching resource #{@url}: #{e}" + nil + end + + private + + def process(url, terminal = false) + @url = url + + perform_request { |response| process_response(response, terminal) } + end + + def perform_request(&block) + Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).perform(&block) + end + + def process_response(response, terminal = false) + return nil if response.code != 200 + + if ['application/activity+json', 'application/ld+json'].include?(response.mime_type) + body = response.body_with_limit + json = body_to_json(body) + + [json['id'], { prefetched_body: body, id: true }, :activitypub] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) || expected_type?(json)) + elsif !terminal + link_header = response['Link'] && parse_link_header(response) + + if link_header&.find_link(%w(rel alternate)) + process_link_headers(link_header) + elsif response.mime_type == 'text/html' + process_html(response) + end + end + end + + def expected_type?(json) + equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) + end + + def process_html(response) + page = Nokogiri::HTML(response.body_with_limit) + json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } + + process(json_link['href'], terminal: true) unless json_link.nil? + end + + def process_link_headers(link_header) + json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) + + process(json_link.href, terminal: true) unless json_link.nil? + end + + def parse_link_header(response) + LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link']) + end +end diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index f941b489a..80381c16b 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -4,64 +4,49 @@ class ResolveURLService < BaseService include JsonLdHelper include Authorization - attr_reader :url - def call(url, on_behalf_of: nil) - @url = url + @url = url @on_behalf_of = on_behalf_of - return process_local_url if local_url? - - process_url unless fetched_atom_feed.nil? + if local_url? + process_local_url + elsif !fetched_resource.nil? + process_url + end end private def process_url if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) - FetchRemoteAccountService.new.call(atom_url, body, protocol) + FetchRemoteAccountService.new.call(resource_url, body, protocol) elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) - FetchRemoteStatusService.new.call(atom_url, body, protocol) + FetchRemoteStatusService.new.call(resource_url, body, protocol) end end - def fetched_atom_feed - @_fetched_atom_feed ||= FetchAtomService.new.call(url) + def fetched_resource + @fetched_resource ||= FetchResourceService.new.call(@url) end - def atom_url - fetched_atom_feed.first + def resource_url + fetched_resource.first end def body - fetched_atom_feed.second[:prefetched_body] + fetched_resource.second[:prefetched_body] end def protocol - fetched_atom_feed.third + fetched_resource.third end def type return json_data['type'] if protocol == :activitypub - - case xml_root - when 'feed' - 'Person' - when 'entry' - 'Note' - end end def json_data - @_json_data ||= body_to_json(body) - end - - def xml_root - xml_data.root.name - end - - def xml_data - @_xml_data ||= Nokogiri::XML(body, nil, 'utf-8') + @json_data ||= body_to_json(body) end def local_url? @@ -83,10 +68,10 @@ class ResolveURLService < BaseService def check_local_status(status) return if status.nil? + authorize_with @on_behalf_of, status, :show? status rescue Mastodon::NotPermittedError - # Do not disclose the existence of status the user is not authorized to see nil end end diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 8b52b8e49..5457d9d4b 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -2,6 +2,7 @@ class ActivityPub::DeliveryWorker include Sidekiq::Worker + include JsonLdHelper STOPLIGHT_FAILURE_THRESHOLD = 10 STOPLIGHT_COOLDOWN = 60 @@ -32,9 +33,10 @@ class ActivityPub::DeliveryWorker private def build_request(http_client) - request = Request.new(:post, @inbox_url, body: @json, http_client: http_client) - request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) - request.add_headers(HEADERS) + Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request| + request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) + request.add_headers(HEADERS) + end end def perform_request @@ -53,14 +55,6 @@ class ActivityPub::DeliveryWorker .run end - def response_successful?(response) - (200...300).cover?(response.code) - end - - def response_error_unsalvageable?(response) - response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code)) - end - def failure_tracker @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url) end diff --git a/spec/services/fetch_atom_service_spec.rb b/spec/services/fetch_atom_service_spec.rb deleted file mode 100644 index 495540004..000000000 --- a/spec/services/fetch_atom_service_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -require 'rails_helper' - -RSpec.describe FetchAtomService, type: :service do - describe '#call' do - let(:url) { 'http://example.com' } - subject { FetchAtomService.new.call(url) } - - context 'url is blank' do - let(:url) { '' } - it { is_expected.to be_nil } - end - - context 'request failed' do - before do - WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {}) - end - - it { is_expected.to be_nil } - end - - context 'raise OpenSSL::SSL::SSLError' do - before do - allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError) - end - - it 'output log and return nil' do - expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('SSL error: OpenSSL::SSL::SSLError') - is_expected.to be_nil - end - end - - context 'raise HTTP::ConnectionError' do - before do - allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError) - end - - it 'output log and return nil' do - expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('HTTP ConnectionError: HTTP::ConnectionError') - is_expected.to be_nil - end - end - - context 'response success' do - let(:body) { '' } - let(:headers) { { 'Content-Type' => content_type } } - let(:json) { - { id: 1, - '@context': ActivityPub::TagManager::CONTEXT, - type: 'Note', - }.to_json - } - - before do - WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) - end - - context 'content type is application/atom+xml' do - let(:content_type) { 'application/atom+xml' } - - it { is_expected.to eq [url, { :prefetched_body => "" }, :ostatus] } - end - - context 'content_type is activity+json' do - let(:content_type) { 'application/activity+json; charset=utf-8' } - let(:body) { json } - - it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } - end - - context 'content_type is ld+json with profile' do - let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } - let(:body) { json } - - it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } - end - - before do - WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) - WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' }) - end - - context 'has link header' do - let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } - - it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } - end - - context 'content type is text/html' do - let(:content_type) { 'text/html' } - let(:body) { '' } - - it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } - end - end - end -end diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb index 37e9910d4..ee7325be2 100644 --- a/spec/services/fetch_remote_account_service_spec.rb +++ b/spec/services/fetch_remote_account_service_spec.rb @@ -4,6 +4,7 @@ RSpec.describe FetchRemoteAccountService, type: :service do let(:url) { 'https://example.com/alice' } let(:prefetched_body) { nil } let(:protocol) { :ostatus } + subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) } let(:actor) do diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb new file mode 100644 index 000000000..17c192c44 --- /dev/null +++ b/spec/services/fetch_resource_service_spec.rb @@ -0,0 +1,96 @@ +require 'rails_helper' + +RSpec.describe FetchResourceService, type: :service do + let!(:representative) { Fabricate(:account) } + + describe '#call' do + let(:url) { 'http://example.com' } + subject { described_class.new.call(url) } + + context 'url is blank' do + let(:url) { '' } + it { is_expected.to be_nil } + end + + context 'request failed' do + before do + WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {}) + end + + it { is_expected.to be_nil } + end + + context 'raise OpenSSL::SSL::SSLError' do + before do + allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError) + end + + it 'return nil' do + is_expected.to be_nil + end + end + + context 'raise HTTP::ConnectionError' do + before do + allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError) + end + + it 'return nil' do + is_expected.to be_nil + end + end + + context 'response success' do + let(:body) { '' } + let(:headers) { { 'Content-Type' => content_type } } + let(:json) { + { id: 1, + '@context': ActivityPub::TagManager::CONTEXT, + type: 'Note', + }.to_json + } + + before do + WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + end + + context 'content type is application/atom+xml' do + let(:content_type) { 'application/atom+xml' } + + it { is_expected.to eq nil } + end + + context 'content_type is activity+json' do + let(:content_type) { 'application/activity+json; charset=utf-8' } + let(:body) { json } + + it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } + end + + context 'content_type is ld+json with profile' do + let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } + let(:body) { json } + + it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } + end + + before do + WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' }) + end + + context 'has link header' do + let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } + + it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } + end + + context 'content type is text/html' do + let(:content_type) { 'text/html' } + let(:body) { '' } + + it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } + end + end + end +end diff --git a/spec/services/resolve_url_service_spec.rb b/spec/services/resolve_url_service_spec.rb index 7bb5d1940..aa4204637 100644 --- a/spec/services/resolve_url_service_spec.rb +++ b/spec/services/resolve_url_service_spec.rb @@ -6,48 +6,14 @@ describe ResolveURLService, type: :service do subject { described_class.new } describe '#call' do - it 'returns nil when there is no atom url' do - url = 'http://example.com/missing-atom' + it 'returns nil when there is no resource url' do + url = 'http://example.com/missing-resource' service = double - allow(FetchAtomService).to receive(:new).and_return service - allow(service).to receive(:call).with(url).and_return(nil) - - result = subject.call(url) - expect(result).to be_nil - end - - it 'fetches remote accounts for feed types' do - url = 'http://example.com/atom-feed' - service = double - allow(FetchAtomService).to receive(:new).and_return service - feed_url = 'http://feed-url' - feed_content = 'contents' - allow(service).to receive(:call).with(url).and_return([feed_url, { prefetched_body: feed_content }]) - - account_service = double - allow(FetchRemoteAccountService).to receive(:new).and_return(account_service) - allow(account_service).to receive(:call) - - _result = subject.call(url) - expect(account_service).to have_received(:call).with(feed_url, feed_content, nil) - end - - it 'fetches remote statuses for entry types' do - url = 'http://example.com/atom-entry' - service = double - allow(FetchAtomService).to receive(:new).and_return service - feed_url = 'http://feed-url' - feed_content = 'contents' - allow(service).to receive(:call).with(url).and_return([feed_url, { prefetched_body: feed_content }]) - - account_service = double - allow(FetchRemoteStatusService).to receive(:new).and_return(account_service) - allow(account_service).to receive(:call) - - _result = subject.call(url) + allow(FetchResourceService).to receive(:new).and_return service + allow(service).to receive(:call).with(url).and_return(nil) - expect(account_service).to have_received(:call).with(feed_url, feed_content, nil) + expect(subject.call(url)).to be_nil end end end -- cgit From 4e8dcc5dbbf625b7268ed10d36122de985da6bdc Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 11 Jul 2019 14:49:55 +0200 Subject: Add HTTP signatures to all outgoing ActivityPub GET requests (#11284) --- app/helpers/jsonld_helper.rb | 13 ++--- app/lib/request.rb | 4 +- app/services/fetch_resource_service.rb | 2 +- .../concerns/signature_verification_spec.rb | 2 +- spec/services/fetch_remote_account_service_spec.rb | 1 + spec/services/fetch_resource_service_spec.rb | 61 +++++++++++++--------- 6 files changed, 43 insertions(+), 40 deletions(-) (limited to 'app/helpers') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 34a657e06..83a5b2462 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -77,19 +77,12 @@ module JsonLdHelper end def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false) - build_request(uri, on_behalf_of).perform do |response| - raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - - return body_to_json(response.body_with_limit) if response.code == 200 - end - - # If request failed, retry without doing it on behalf of a user - return if on_behalf_of.nil? + on_behalf_of ||= Account.representative - build_request(uri).perform do |response| + build_request(uri, on_behalf_of).perform do |response| raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - response.code == 200 ? body_to_json(response.body_with_limit) : nil + body_to_json(response.body_with_limit) if response.code == 200 end end diff --git a/app/lib/request.rb b/app/lib/request.rb index 1fd3f5190..9d874fe2c 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -40,8 +40,8 @@ class Request set_digest! if options.key?(:body) end - def on_behalf_of(account, key_id_format = :acct, sign_with: nil) - raise ArgumentError, 'account must be local' unless account&.local? + def on_behalf_of(account, key_id_format = :uri, sign_with: nil) + raise ArgumentError, 'account must not be nil' if account.nil? @account = account @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb index c0473f3ad..3676d899d 100644 --- a/app/services/fetch_resource_service.rb +++ b/app/services/fetch_resource_service.rb @@ -23,7 +23,7 @@ class FetchResourceService < BaseService end def perform_request(&block) - Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).perform(&block) + Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).on_behalf_of(Account.representative).perform(&block) end def process_response(response, terminal = false) diff --git a/spec/controllers/concerns/signature_verification_spec.rb b/spec/controllers/concerns/signature_verification_spec.rb index 720690097..1fa19f54d 100644 --- a/spec/controllers/concerns/signature_verification_spec.rb +++ b/spec/controllers/concerns/signature_verification_spec.rb @@ -38,7 +38,7 @@ describe ApplicationController, type: :controller do end context 'with signature header' do - let!(:author) { Fabricate(:account) } + let!(:author) { Fabricate(:account, domain: 'example.com', uri: 'https://example.com/actor') } context 'without body' do before do diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb index ee7325be2..b37445861 100644 --- a/spec/services/fetch_remote_account_service_spec.rb +++ b/spec/services/fetch_remote_account_service_spec.rb @@ -4,6 +4,7 @@ RSpec.describe FetchRemoteAccountService, type: :service do let(:url) { 'https://example.com/alice' } let(:prefetched_body) { nil } let(:protocol) { :ostatus } + let!(:representative) { Fabricate(:account) } subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) } diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb index 17c192c44..98630966b 100644 --- a/spec/services/fetch_resource_service_spec.rb +++ b/spec/services/fetch_resource_service_spec.rb @@ -5,69 +5,78 @@ RSpec.describe FetchResourceService, type: :service do describe '#call' do let(:url) { 'http://example.com' } + subject { described_class.new.call(url) } - context 'url is blank' do + context 'with blank url' do let(:url) { '' } it { is_expected.to be_nil } end - context 'request failed' do + context 'when request fails' do before do - WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {}) + stub_request(:get, url).to_return(status: 500, body: '', headers: {}) end it { is_expected.to be_nil } end - context 'raise OpenSSL::SSL::SSLError' do + context 'when OpenSSL::SSL::SSLError is raised' do before do - allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError) + allow(Request).to receive_message_chain(:new, :add_headers, :on_behalf_of, :perform).and_raise(OpenSSL::SSL::SSLError) end - it 'return nil' do - is_expected.to be_nil - end + it { is_expected.to be_nil } end - context 'raise HTTP::ConnectionError' do + context 'when HTTP::ConnectionError is raised' do before do - allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError) + allow(Request).to receive_message_chain(:new, :add_headers, :on_behalf_of, :perform).and_raise(HTTP::ConnectionError) end - it 'return nil' do - is_expected.to be_nil - end + it { is_expected.to be_nil } end - context 'response success' do + context 'when request succeeds' do let(:body) { '' } - let(:headers) { { 'Content-Type' => content_type } } - let(:json) { - { id: 1, + + let(:content_type) { 'application/json' } + + let(:headers) do + { 'Content-Type' => content_type } + end + + let(:json) do + { + id: 1, '@context': ActivityPub::TagManager::CONTEXT, type: 'Note', }.to_json - } + end before do - WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + end + + it 'signs request' do + subject + expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(representative) + '#main-key')}"/ })).to have_been_made end - context 'content type is application/atom+xml' do + context 'when content type is application/atom+xml' do let(:content_type) { 'application/atom+xml' } it { is_expected.to eq nil } end - context 'content_type is activity+json' do + context 'when content type is activity+json' do let(:content_type) { 'application/activity+json; charset=utf-8' } let(:body) { json } it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] } end - context 'content_type is ld+json with profile' do + context 'when content type is ld+json with profile' do let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } let(:body) { json } @@ -75,17 +84,17 @@ RSpec.describe FetchResourceService, type: :service do end before do - WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers) - WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' }) + stub_request(:get, url).to_return(status: 200, body: body, headers: headers) + stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' }) end - context 'has link header' do + context 'when link header is present' do let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] } end - context 'content type is text/html' do + context 'when content type is text/html' do let(:content_type) { 'text/html' } let(:body) { '' } -- cgit