Add synchronization of remote featured tags (#19380)

* Add LIMIT of featured tag to instance API response

* Add featured_tags_collection_url to Account

* Add synchronization of remote featured tags

* Deliver update activity when updating featured tag

* Remove featured_tags_collection_url

* Revert "Add featured_tags_collection_url to Account"

This reverts commit cff349fc27b104ded2df6bb5665132dc24dab09c.

* Add hashtag sync from featured collections

* Fix tag name normalize

* Add target option to fetch featured collection

* Refactor fetch_featured_tags_collection_service

* Add LIMIT of featured tag to v1/instance API response

11 files changed, 148 insertions, 5 deletions

diff --git a/app/controllers/api/v1/featured_tags_controller.rb b/app/controllers/api/v1/featured_tags_controller.rb
index c1ead4f54..a67db7040 100644
--- a/app/controllers/api/v1/featured_tags_controller.rb
+++ b/app/controllers/api/v1/featured_tags_controller.rb
@@ -14,11 +14,13 @@ class Api::V1::FeaturedTagsController < Api::BaseController
 
   def create
     @featured_tag = current_account.featured_tags.create!(featured_tag_params)
+    ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id)
     render json: @featured_tag, serializer: REST::FeaturedTagSerializer
   end
 
   def destroy
     @featured_tag.destroy!
+    ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id)
     render_empty
   end
 
diff --git a/app/controllers/settings/featured_tags_controller.rb b/app/controllers/settings/featured_tags_controller.rb
index aadff7c83..ae714e912 100644
--- a/app/controllers/settings/featured_tags_controller.rb
+++ b/app/controllers/settings/featured_tags_controller.rb
@@ -13,6 +13,7 @@ class Settings::FeaturedTagsController < Settings::BaseController
     @featured_tag = current_account.featured_tags.new(featured_tag_params)
 
     if @featured_tag.save
+      ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id)
       redirect_to settings_featured_tags_path
     else
       set_featured_tags
@@ -24,6 +25,7 @@ class Settings::FeaturedTagsController < Settings::BaseController
 
   def destroy
     @featured_tag.destroy!
+    ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id)
     redirect_to settings_featured_tags_path
   end
 
diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb
index 201ce75f5..b16c79ac7 100644
--- a/app/models/featured_tag.rb
+++ b/app/models/featured_tag.rb
@@ -26,6 +26,8 @@ class FeaturedTag < ApplicationRecord
 
   attr_writer :name
 
+  LIMIT = 10
+
   def name
     tag_id.present? ? tag.name : @name
   end
@@ -50,7 +52,7 @@ class FeaturedTag < ApplicationRecord
   end
 
   def validate_featured_tags_limit
-    errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= 10
+    errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT
   end
 
   def validate_tag_name
diff --git a/app/serializers/rest/instance_serializer.rb b/app/serializers/rest/instance_serializer.rb
index dfa8ce40a..7d00b20ba 100644
--- a/app/serializers/rest/instance_serializer.rb
+++ b/app/serializers/rest/instance_serializer.rb
@@ -47,6 +47,10 @@ class REST::InstanceSerializer < ActiveModel::Serializer
         streaming: Rails.configuration.x.streaming_api_base_url,
       },
 
+      accounts: {
+        max_featured_tags: FeaturedTag::LIMIT,
+      },
+
       statuses: {
         max_characters: StatusLengthValidator::MAX_CHARS,
         max_media_attachments: 4,
diff --git a/app/serializers/rest/v1/instance_serializer.rb b/app/serializers/rest/v1/instance_serializer.rb
index 872175451..99d1b2bd6 100644
--- a/app/serializers/rest/v1/instance_serializer.rb
+++ b/app/serializers/rest/v1/instance_serializer.rb
@@ -58,6 +58,10 @@ class REST::V1::InstanceSerializer < ActiveModel::Serializer
 
   def configuration
     {
+      accounts: {
+        max_featured_tags: FeaturedTag::LIMIT,
+      },
+
       statuses: {
         max_characters: StatusLengthValidator::MAX_CHARS,
         max_media_attachments: 4,
diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index 37d05e055..026fe24c5 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -3,10 +3,11 @@
 class ActivityPub::FetchFeaturedCollectionService < BaseService
   include JsonLdHelper
 
-  def call(account)
+  def call(account, **options)
     return if account.featured_collection_url.blank? || account.suspended? || account.local?
 
     @account = account
+    @options = options
     @json    = fetch_resource(@account.featured_collection_url, true, local_follower)
 
     return unless supported_context?(@json)
@@ -36,7 +37,15 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
   end
 
   def process_items(items)
+    process_note_items(items) if @options[:note]
+    process_hashtag_items(items) if @options[:hashtag]
+  end
+
+  def process_note_items(items)
     status_ids = items.filter_map do |item|
+      type = item['type']
+      next unless type == 'Note'
+
       uri = value_or_id(item)
       next if ActivityPub::TagManager.instance.local_uri?(uri)
 
@@ -67,6 +76,26 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
     end
   end
 
+  def process_hashtag_items(items)
+    names     = items.filter_map { |item| item['type'] == 'Hashtag' && item['name']&.delete_prefix('#') }.map { |name| HashtagNormalizer.new.normalize(name) }
+    to_remove = []
+    to_add    = names
+
+    FeaturedTag.where(account: @account).map(&:name).each do |name|
+      if names.include?(name)
+        to_add.delete(name)
+      else
+        to_remove << name
+      end
+    end
+
+    FeaturedTag.includes(:tag).where(account: @account, tags: { name: to_remove }).delete_all unless to_remove.empty?
+
+    to_add.each do |name|
+      FeaturedTag.create!(account: @account, name: name)
+    end
+  end
+
   def local_follower
     return @local_follower if defined?(@local_follower)
 
diff --git a/app/services/activitypub/fetch_featured_tags_collection_service.rb b/app/services/activitypub/fetch_featured_tags_collection_service.rb
new file mode 100644
index 000000000..555919938
--- /dev/null
+++ b/app/services/activitypub/fetch_featured_tags_collection_service.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class ActivityPub::FetchFeaturedTagsCollectionService < BaseService
+  include JsonLdHelper
+
+  def call(account, url)
+    return if url.blank? || account.suspended? || account.local?
+
+    @account = account
+    @json    = fetch_resource(url, true, local_follower)
+
+    return unless supported_context?(@json)
+
+    process_items(collection_items(@json))
+  end
+
+  private
+
+  def collection_items(collection)
+    all_items = []
+
+    collection = fetch_collection(collection['first']) if collection['first'].present?
+
+    while collection.is_a?(Hash)
+      items = begin
+        case collection['type']
+        when 'Collection', 'CollectionPage'
+          collection['items']
+        when 'OrderedCollection', 'OrderedCollectionPage'
+          collection['orderedItems']
+        end
+      end
+
+      break if items.blank?
+
+      all_items.concat(items)
+
+      break if all_items.size >= FeaturedTag::LIMIT
+
+      collection = collection['next'].present? ? fetch_collection(collection['next']) : nil
+    end
+
+    all_items
+  end
+
+  def fetch_collection(collection_or_uri)
+    return collection_or_uri if collection_or_uri.is_a?(Hash)
+    return if invalid_origin?(collection_or_uri)
+
+    fetch_resource_without_id_validation(collection_or_uri, local_follower, true)
+  end
+
+  def process_items(items)
+    names     = items.filter_map { |item| item['type'] == 'Hashtag' && item['name']&.delete_prefix('#') }.map { |name| HashtagNormalizer.new.normalize(name) }
+    to_remove = []
+    to_add    = names
+
+    FeaturedTag.where(account: @account).map(&:name).each do |name|
+      if names.include?(name)
+        to_add.delete(name)
+      else
+        to_remove << name
+      end
+    end
+
+    FeaturedTag.includes(:tag).where(account: @account, tags: { name: to_remove }).delete_all unless to_remove.empty?
+
+    to_add.each do |name|
+      FeaturedTag.create!(account: @account, name: name)
+    end
+  end
+
+  def local_follower
+    return @local_follower if defined?(@local_follower)
+
+    @local_follower = @account.followers.local.without_suspended.first
+  end
+end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 456b3524b..3834d79cc 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -39,6 +39,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
     unless @options[:only_key] || @account.suspended?
       check_featured_collection! if @account.featured_collection_url.present?
+      check_featured_tags_collection! if @json['featuredTags'].present?
       check_links! unless @account.fields.empty?
     end
 
@@ -149,7 +150,11 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def check_featured_collection!
-    ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id)
+    ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'hashtag' => @json['featuredTags'].blank? })
+  end
+
+  def check_featured_tags_collection!
+    ActivityPub::SynchronizeFeaturedTagsCollectionWorker.perform_async(@account.id, @json['featuredTags'])
   end
 
   def check_links!
diff --git a/app/workers/activitypub/synchronize_featured_collection_worker.rb b/app/workers/activitypub/synchronize_featured_collection_worker.rb
index 7a0898e89..f67d693cb 100644
--- a/app/workers/activitypub/synchronize_featured_collection_worker.rb
+++ b/app/workers/activitypub/synchronize_featured_collection_worker.rb
@@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker
 
   sidekiq_options queue: 'pull', lock: :until_executed
 
-  def perform(account_id)
-    ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id))
+  def perform(account_id, options = {})
+    options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys)
+
+    ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb
new file mode 100644
index 000000000..14af4f725
--- /dev/null
+++ b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class ActivityPub::SynchronizeFeaturedTagsCollectionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull', lock: :until_executed
+
+  def perform(account_id, url)
+    ActivityPub::FetchFeaturedTagsCollectionService.new.call(Account.find(account_id), url)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb
index 81fde63b8..d0391bb6f 100644
--- a/app/workers/activitypub/update_distribution_worker.rb
+++ b/app/workers/activitypub/update_distribution_worker.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker
+  sidekiq_options queue: 'push', lock: :until_executed
+
   # Distribute an profile update to servers that might have a copy
   # of the account in question
   def perform(account_id, options = {})