From b0e3f0312c3271a2705f912602fcba70f4ed8b69 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 20 Oct 2022 16:15:52 +0900 Subject: Add synchronization of remote featured tags (#19380) * Add LIMIT of featured tag to instance API response * Add featured_tags_collection_url to Account * Add synchronization of remote featured tags * Deliver update activity when updating featured tag * Remove featured_tags_collection_url * Revert "Add featured_tags_collection_url to Account" This reverts commit cff349fc27b104ded2df6bb5665132dc24dab09c. * Add hashtag sync from featured collections * Fix tag name normalize * Add target option to fetch featured collection * Refactor fetch_featured_tags_collection_service * Add LIMIT of featured tag to v1/instance API response --- .../activitypub/synchronize_featured_collection_worker.rb | 6 ++++-- .../synchronize_featured_tags_collection_worker.rb | 13 +++++++++++++ app/workers/activitypub/update_distribution_worker.rb | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 app/workers/activitypub/synchronize_featured_tags_collection_worker.rb (limited to 'app/workers') diff --git a/app/workers/activitypub/synchronize_featured_collection_worker.rb b/app/workers/activitypub/synchronize_featured_collection_worker.rb index 7a0898e89..f67d693cb 100644 --- a/app/workers/activitypub/synchronize_featured_collection_worker.rb +++ b/app/workers/activitypub/synchronize_featured_collection_worker.rb @@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker sidekiq_options queue: 'pull', lock: :until_executed - def perform(account_id) - ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id)) + def perform(account_id, options = {}) + options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys) + + ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options) rescue ActiveRecord::RecordNotFound true end diff --git a/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb new file mode 100644 index 000000000..14af4f725 --- /dev/null +++ b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ActivityPub::SynchronizeFeaturedTagsCollectionWorker + include Sidekiq::Worker + + sidekiq_options queue: 'pull', lock: :until_executed + + def perform(account_id, url) + ActivityPub::FetchFeaturedTagsCollectionService.new.call(Account.find(account_id), url) + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb index 81fde63b8..d0391bb6f 100644 --- a/app/workers/activitypub/update_distribution_worker.rb +++ b/app/workers/activitypub/update_distribution_worker.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker + sidekiq_options queue: 'push', lock: :until_executed + # Distribute an profile update to servers that might have a copy # of the account in question def perform(account_id, options = {}) -- cgit From 74ead7d10698c7f18ca22c07d2f04ff81f419097 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Sun, 23 Oct 2022 01:30:55 +0900 Subject: Change featured tag updates to add/remove activity (#19409) * Change featured tag updates to add/remove activity * Fix to check for the existence of feature tag * Rename service and worker * Merge AddHashtagSerializer with AddSerializer * Undo removal of sidekiq_options --- app/controllers/api/v1/featured_tags_controller.rb | 8 +++----- .../settings/featured_tags_controller.rb | 13 ++++++------ app/models/featured_tag.rb | 4 ++++ app/serializers/activitypub/add_serializer.rb | 23 ++++++++++++++++++++-- app/serializers/activitypub/hashtag_serializer.rb | 2 ++ app/serializers/activitypub/remove_serializer.rb | 23 ++++++++++++++++++++-- app/services/create_featured_tag_service.rb | 21 ++++++++++++++++++++ app/services/remove_featured_tag_service.rb | 18 +++++++++++++++++ .../activitypub/account_raw_distribution_worker.rb | 9 +++++++++ app/workers/remove_featured_tag_worker.rb | 11 +++++++++++ 10 files changed, 117 insertions(+), 15 deletions(-) create mode 100644 app/services/create_featured_tag_service.rb create mode 100644 app/services/remove_featured_tag_service.rb create mode 100644 app/workers/activitypub/account_raw_distribution_worker.rb create mode 100644 app/workers/remove_featured_tag_worker.rb (limited to 'app/workers') diff --git a/app/controllers/api/v1/featured_tags_controller.rb b/app/controllers/api/v1/featured_tags_controller.rb index a67db7040..edb42a94e 100644 --- a/app/controllers/api/v1/featured_tags_controller.rb +++ b/app/controllers/api/v1/featured_tags_controller.rb @@ -13,14 +13,12 @@ class Api::V1::FeaturedTagsController < Api::BaseController end def create - @featured_tag = current_account.featured_tags.create!(featured_tag_params) - ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) - render json: @featured_tag, serializer: REST::FeaturedTagSerializer + featured_tag = CreateFeaturedTagService.new.call(current_account, featured_tag_params[:name]) + render json: featured_tag, serializer: REST::FeaturedTagSerializer end def destroy - @featured_tag.destroy! - ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) + RemoveFeaturedTagWorker.perform_async(current_account.id, @featured_tag.id) render_empty end diff --git a/app/controllers/settings/featured_tags_controller.rb b/app/controllers/settings/featured_tags_controller.rb index ae714e912..cc6ec0843 100644 --- a/app/controllers/settings/featured_tags_controller.rb +++ b/app/controllers/settings/featured_tags_controller.rb @@ -10,10 +10,8 @@ class Settings::FeaturedTagsController < Settings::BaseController end def create - @featured_tag = current_account.featured_tags.new(featured_tag_params) - - if @featured_tag.save - ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) + if !featured_tag_exists? + CreateFeaturedTagService.new.call(current_account, featured_tag_params[:name]) redirect_to settings_featured_tags_path else set_featured_tags @@ -24,13 +22,16 @@ class Settings::FeaturedTagsController < Settings::BaseController end def destroy - @featured_tag.destroy! - ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) + RemoveFeaturedTagWorker.perform_async(current_account.id, @featured_tag.id) redirect_to settings_featured_tags_path end private + def featured_tag_exists? + current_account.featured_tags.by_name(featured_tag_params[:name]).exists? + end + def set_featured_tag @featured_tag = current_account.featured_tags.find(params[:id]) end diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb index ec234a6fd..3f5cddce6 100644 --- a/app/models/featured_tag.rb +++ b/app/models/featured_tag.rb @@ -30,6 +30,10 @@ class FeaturedTag < ApplicationRecord LIMIT = 10 + def sign? + true + end + def name tag_id.present? ? tag.name : @name end diff --git a/app/serializers/activitypub/add_serializer.rb b/app/serializers/activitypub/add_serializer.rb index 6f5aab17f..436b05086 100644 --- a/app/serializers/activitypub/add_serializer.rb +++ b/app/serializers/activitypub/add_serializer.rb @@ -1,10 +1,29 @@ # frozen_string_literal: true class ActivityPub::AddSerializer < ActivityPub::Serializer + class UriSerializer < ActiveModel::Serializer + include RoutingHelper + + def serializable_hash(*_args) + ActivityPub::TagManager.instance.uri_for(object) + end + end + + def self.serializer_for(model, options) + case model.class.name + when 'Status' + UriSerializer + when 'FeaturedTag' + ActivityPub::HashtagSerializer + else + super + end + end + include RoutingHelper attributes :type, :actor, :target - attribute :proper_object, key: :object + has_one :proper_object, key: :object def type 'Add' @@ -15,7 +34,7 @@ class ActivityPub::AddSerializer < ActivityPub::Serializer end def proper_object - ActivityPub::TagManager.instance.uri_for(object) + object end def target diff --git a/app/serializers/activitypub/hashtag_serializer.rb b/app/serializers/activitypub/hashtag_serializer.rb index 90929c57f..2b24eb8cc 100644 --- a/app/serializers/activitypub/hashtag_serializer.rb +++ b/app/serializers/activitypub/hashtag_serializer.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::HashtagSerializer < ActivityPub::Serializer + context_extensions :hashtag + include RoutingHelper attributes :type, :href, :name diff --git a/app/serializers/activitypub/remove_serializer.rb b/app/serializers/activitypub/remove_serializer.rb index 7fefda59d..fb224f8a9 100644 --- a/app/serializers/activitypub/remove_serializer.rb +++ b/app/serializers/activitypub/remove_serializer.rb @@ -1,10 +1,29 @@ # frozen_string_literal: true class ActivityPub::RemoveSerializer < ActivityPub::Serializer + class UriSerializer < ActiveModel::Serializer + include RoutingHelper + + def serializable_hash(*_args) + ActivityPub::TagManager.instance.uri_for(object) + end + end + + def self.serializer_for(model, options) + case model.class.name + when 'Status' + UriSerializer + when 'FeaturedTag' + ActivityPub::HashtagSerializer + else + super + end + end + include RoutingHelper attributes :type, :actor, :target - attribute :proper_object, key: :object + has_one :proper_object, key: :object def type 'Remove' @@ -15,7 +34,7 @@ class ActivityPub::RemoveSerializer < ActivityPub::Serializer end def proper_object - ActivityPub::TagManager.instance.uri_for(object) + object end def target diff --git a/app/services/create_featured_tag_service.rb b/app/services/create_featured_tag_service.rb new file mode 100644 index 000000000..c99d16113 --- /dev/null +++ b/app/services/create_featured_tag_service.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class CreateFeaturedTagService < BaseService + include Payloadable + + def call(account, name) + @account = account + + FeaturedTag.create!(account: account, name: name).tap do |featured_tag| + ActivityPub::AccountRawDistributionWorker.perform_async(build_json(featured_tag), account.id) if @account.local? + end + rescue ActiveRecord::RecordNotUnique + FeaturedTag.by_name(name).find_by!(account: account) + end + + private + + def build_json(featured_tag) + Oj.dump(serialize_payload(featured_tag, ActivityPub::AddSerializer, signer: @account)) + end +end diff --git a/app/services/remove_featured_tag_service.rb b/app/services/remove_featured_tag_service.rb new file mode 100644 index 000000000..2aa70e8fc --- /dev/null +++ b/app/services/remove_featured_tag_service.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class RemoveFeaturedTagService < BaseService + include Payloadable + + def call(account, featured_tag) + @account = account + + featured_tag.destroy! + ActivityPub::AccountRawDistributionWorker.perform_async(build_json(featured_tag), account.id) if @account.local? + end + + private + + def build_json(featured_tag) + Oj.dump(serialize_payload(featured_tag, ActivityPub::RemoveSerializer, signer: @account)) + end +end diff --git a/app/workers/activitypub/account_raw_distribution_worker.rb b/app/workers/activitypub/account_raw_distribution_worker.rb new file mode 100644 index 000000000..a84c7d214 --- /dev/null +++ b/app/workers/activitypub/account_raw_distribution_worker.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class ActivityPub::AccountRawDistributionWorker < ActivityPub::RawDistributionWorker + protected + + def inboxes + @inboxes ||= AccountReachFinder.new(@account).inboxes + end +end diff --git a/app/workers/remove_featured_tag_worker.rb b/app/workers/remove_featured_tag_worker.rb new file mode 100644 index 000000000..065ec79d8 --- /dev/null +++ b/app/workers/remove_featured_tag_worker.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RemoveFeaturedTagWorker + include Sidekiq::Worker + + def perform(account_id, featured_tag_id) + RemoveFeaturedTagService.new.call(Account.find(account_id), FeaturedTag.find(featured_tag_id)) + rescue ActiveRecord::RecordNotFound + true + end +end -- cgit From bf0ab3e0fac54515c13beef4ec09b0455f1bce67 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 26 Oct 2022 12:10:48 +0200 Subject: Fix vacuum scheduler missing lock, locks never expiring (#19458) Remove vacuuming of orphaned preview cards --- app/lib/vacuum/media_attachments_vacuum.rb | 2 +- app/lib/vacuum/preview_cards_vacuum.rb | 9 --------- app/workers/scheduler/vacuum_scheduler.rb | 2 +- config/initializers/sidekiq.rb | 1 + spec/lib/vacuum/preview_cards_vacuum_spec.rb | 4 ---- 5 files changed, 3 insertions(+), 15 deletions(-) (limited to 'app/workers') diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb index 7fb347ce4..7c0a85a9d 100644 --- a/app/lib/vacuum/media_attachments_vacuum.rb +++ b/app/lib/vacuum/media_attachments_vacuum.rb @@ -8,8 +8,8 @@ class Vacuum::MediaAttachmentsVacuum end def perform - vacuum_cached_files! if retention_period? vacuum_orphaned_records! + vacuum_cached_files! if retention_period? end private diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb index 84ef100ed..14fdeda1c 100644 --- a/app/lib/vacuum/preview_cards_vacuum.rb +++ b/app/lib/vacuum/preview_cards_vacuum.rb @@ -9,7 +9,6 @@ class Vacuum::PreviewCardsVacuum def perform vacuum_cached_images! if retention_period? - vacuum_orphaned_records! end private @@ -21,18 +20,10 @@ class Vacuum::PreviewCardsVacuum end end - def vacuum_orphaned_records! - orphaned_preview_cards.in_batches.destroy_all - end - def preview_cards_past_retention_period PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago)) end - def orphaned_preview_cards - PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago)) - end - def retention_period? @retention_period.present? end diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb index ce88ff204..9544f808b 100644 --- a/app/workers/scheduler/vacuum_scheduler.rb +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::VacuumScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed def perform vacuum_operations.each do |operation| diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index c1327053d..9d2abf074 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -35,4 +35,5 @@ SidekiqUniqueJobs.configure do |config| config.reaper_count = 1000 config.reaper_interval = 600 config.reaper_timeout = 150 + config.lock_ttl = 50.days.to_i end diff --git a/spec/lib/vacuum/preview_cards_vacuum_spec.rb b/spec/lib/vacuum/preview_cards_vacuum_spec.rb index 4a4a599fa..275f9ba92 100644 --- a/spec/lib/vacuum/preview_cards_vacuum_spec.rb +++ b/spec/lib/vacuum/preview_cards_vacuum_spec.rb @@ -28,9 +28,5 @@ RSpec.describe Vacuum::PreviewCardsVacuum do it 'does not delete attached preview cards' do expect(new_preview_card.reload).to be_persisted end - - it 'deletes preview cards not attached to any status' do - expect { orphaned_preview_card.reload }.to raise_error ActiveRecord::RecordNotFound - end end end -- cgit