From 9660aa4543deff41c60d131e081137f84e771499 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 8 Mar 2020 23:56:18 +0100 Subject: Change local media attachments to perform heavy processing asynchronously (#13210) Fix #9106 --- app/services/post_status_service.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/services') diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 2301621ab..c61b3baa2 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -101,6 +101,7 @@ class PostStatusService < BaseService @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i)) raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?) + raise Mastodon::ValidationError, I18n.t('media_attachments.validations.not_ready') if @media.any?(&:not_processed?) end def language_from_option(str) -- cgit From b154428e14861f5cdc7ba6e5f8e582dbf7d0a1c0 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 9 Mar 2020 00:10:29 +0100 Subject: Add federation support for the "hide network" preference (#11673) * Change ActivityPub follower/following collections to not link first page * Add support for hiding followers and following of remote users * Switch to using a single `hide_collections` column * Address code style remarks --- .../v1/accounts/follower_accounts_controller.rb | 2 +- .../v1/accounts/following_accounts_controller.rb | 2 +- app/controllers/follower_accounts_controller.rb | 11 +++++++++- app/controllers/following_accounts_controller.rb | 11 +++++++++- app/models/account.rb | 9 ++++++++ .../activitypub/process_account_service.rb | 25 ++++++++++++++++------ ...91212163405_add_hide_collections_to_accounts.rb | 5 +++++ db/schema.rb | 1 + 8 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 db/migrate/20191212163405_add_hide_collections_to_accounts.rb (limited to 'app/services') diff --git a/app/controllers/api/v1/accounts/follower_accounts_controller.rb b/app/controllers/api/v1/accounts/follower_accounts_controller.rb index 850702cca..1daa1ed0d 100644 --- a/app/controllers/api/v1/accounts/follower_accounts_controller.rb +++ b/app/controllers/api/v1/accounts/follower_accounts_controller.rb @@ -25,7 +25,7 @@ class Api::V1::Accounts::FollowerAccountsController < Api::BaseController end def hide_results? - (@account.user_hides_network? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account)) + (@account.hides_followers? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account)) end def default_accounts diff --git a/app/controllers/api/v1/accounts/following_accounts_controller.rb b/app/controllers/api/v1/accounts/following_accounts_controller.rb index 830dcd8a1..6fc23cf75 100644 --- a/app/controllers/api/v1/accounts/following_accounts_controller.rb +++ b/app/controllers/api/v1/accounts/following_accounts_controller.rb @@ -25,7 +25,7 @@ class Api::V1::Accounts::FollowingAccountsController < Api::BaseController end def hide_results? - (@account.user_hides_network? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account)) + (@account.hides_following? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account)) end def default_accounts diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb index 7103749ad..14e22dd1e 100644 --- a/app/controllers/follower_accounts_controller.rb +++ b/app/controllers/follower_accounts_controller.rb @@ -28,7 +28,8 @@ class FollowerAccountsController < ApplicationController render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, - content_type: 'application/activity+json' + content_type: 'application/activity+json', + fields: restrict_fields_to end end end @@ -71,4 +72,12 @@ class FollowerAccountsController < ApplicationController ) end end + + def restrict_fields_to + if page_requested? || !@account.user_hides_network? + # Return all fields + else + %i(id type totalItems) + end + end end diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb index 6c8fb84d8..95849ffb9 100644 --- a/app/controllers/following_accounts_controller.rb +++ b/app/controllers/following_accounts_controller.rb @@ -28,7 +28,8 @@ class FollowingAccountsController < ApplicationController render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, - content_type: 'application/activity+json' + content_type: 'application/activity+json', + fields: restrict_fields_to end end end @@ -71,4 +72,12 @@ class FollowingAccountsController < ApplicationController ) end end + + def restrict_fields_to + if page_requested? || !@account.user_hides_network? + # Return all fields + else + %i(id type totalItems) + end + end end diff --git a/app/models/account.rb b/app/models/account.rb index a1b4a065b..6aceb809c 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -46,6 +46,7 @@ # silenced_at :datetime # suspended_at :datetime # trust_level :integer +# hide_collections :boolean # class Account < ApplicationRecord @@ -323,6 +324,14 @@ class Account < ApplicationRecord save! end + def hides_followers? + hide_collections? || user_hides_network? + end + + def hides_following? + hide_collections? || user_hides_network? + end + def object_type :person end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index d5ede0388..7b4c53d50 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -94,6 +94,7 @@ class ActivityPub::ProcessAccountService < BaseService @account.statuses_count = outbox_total_items if outbox_total_items.present? @account.following_count = following_total_items if following_total_items.present? @account.followers_count = followers_total_items if followers_total_items.present? + @account.hide_collections = following_private? || followers_private? @account.moved_to_account = @json['movedTo'].present? ? moved_account : nil end @@ -166,26 +167,36 @@ class ActivityPub::ProcessAccountService < BaseService end def outbox_total_items - collection_total_items('outbox') + collection_info('outbox').first end def following_total_items - collection_total_items('following') + collection_info('following').first end def followers_total_items - collection_total_items('followers') + collection_info('followers').first end - def collection_total_items(type) - return if @json[type].blank? + def following_private? + !collection_info('following').last + end + + def followers_private? + !collection_info('followers').last + end + + def collection_info(type) + return [nil, nil] if @json[type].blank? return @collections[type] if @collections.key?(type) collection = fetch_resource_without_id_validation(@json[type]) - @collections[type] = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil + total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil + has_first_page = collection.is_a?(Hash) && collection['first'].present? + @collections[type] = [total_items, has_first_page] rescue HTTP::Error, OpenSSL::SSL::SSLError - @collections[type] = nil + @collections[type] = [nil, nil] end def moved_account diff --git a/db/migrate/20191212163405_add_hide_collections_to_accounts.rb b/db/migrate/20191212163405_add_hide_collections_to_accounts.rb new file mode 100644 index 000000000..fa99b32e5 --- /dev/null +++ b/db/migrate/20191212163405_add_hide_collections_to_accounts.rb @@ -0,0 +1,5 @@ +class AddHideCollectionsToAccounts < ActiveRecord::Migration[5.2] + def change + add_column :accounts, :hide_collections, :boolean + end +end diff --git a/db/schema.rb b/db/schema.rb index ddb9a5d8c..a851e01fc 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -170,6 +170,7 @@ ActiveRecord::Schema.define(version: 2020_03_06_035625) do t.datetime "silenced_at" t.datetime "suspended_at" t.integer "trust_level" + t.boolean "hide_collections" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id" -- cgit From cb12a2cdd3fbe2c07a556610596a2de5446b1f50 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 12 Mar 2020 23:06:43 +0100 Subject: Fix some timeouts when searching URLs by limiting some database queries (#13253) Only look up private toots from database if the request failed because of 401, 403 or 404 errors, as those may indicate a private toot, rather than something that isn't a toot or cannot be processed. --- app/services/fetch_resource_service.rb | 3 +++ app/services/resolve_url_service.rb | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'app/services') diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb index abe7766d4..880cdde92 100644 --- a/app/services/fetch_resource_service.rb +++ b/app/services/fetch_resource_service.rb @@ -5,6 +5,8 @@ class FetchResourceService < BaseService ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html;q=0.1' + attr_reader :response_code + def call(url) return if url.blank? @@ -27,6 +29,7 @@ class FetchResourceService < BaseService end def process_response(response, terminal = false) + @response_code = response.code return nil if response.code != 200 if ['application/activity+json', 'application/ld+json'].include?(response.mime_type) diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index 1a2b0d60c..78080d878 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -12,7 +12,7 @@ class ResolveURLService < BaseService process_local_url elsif !fetched_resource.nil? process_url - elsif @on_behalf_of.present? + else process_url_from_db end end @@ -30,6 +30,8 @@ class ResolveURLService < BaseService end def process_url_from_db + return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code) + # It may happen that the resource is a private toot, and thus not fetchable, # but we can return the toot if we already know about it. status = Status.find_by(uri: @url) || Status.find_by(url: @url) @@ -40,7 +42,11 @@ class ResolveURLService < BaseService end def fetched_resource - @fetched_resource ||= FetchResourceService.new.call(@url) + @fetched_resource ||= fetch_resource_service.call(@url) + end + + def fetch_resource_service + @_fetch_resource_service ||= FetchResourceService.new end def resource_url -- cgit