From 619fad6cf8078ea997554081febe850404bee73c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 11 Apr 2021 11:22:50 +0200
Subject: Remove spam check and dependency on nilsimsa gem (#16011)

---
 app/services/process_mentions_service.rb | 5 -----
 app/services/remove_status_service.rb    | 5 -----
 2 files changed, 10 deletions(-)

(limited to 'app/services')

diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 4c02c7865..73dbb1834 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -43,7 +43,6 @@ class ProcessMentionsService < BaseService
     end
 
     status.save!
-    check_for_spam(status)
 
     mentions.each { |mention| create_notification(mention) }
   end
@@ -72,8 +71,4 @@ class ProcessMentionsService < BaseService
   def resolve_account_service
     ResolveAccountService.new
   end
-
-  def check_for_spam(status)
-    SpamCheck.perform(status)
-  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index d6043fb5d..d642beeaa 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -41,7 +41,6 @@ class RemoveStatusService < BaseService
           remove_from_hashtags
           remove_from_public
           remove_from_media if @status.media_attachments.any?
-          remove_from_spam_check
           remove_media
         end
 
@@ -163,10 +162,6 @@ class RemoveStatusService < BaseService
     @status.media_attachments.destroy_all
   end
 
-  def remove_from_spam_check
-    redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
-  end
-
   def lock_options
     { redis: Redis.current, key: "distribute:#{@status.id}" }
   end
-- 
cgit 


From dde8739020eb201d1c9a31da116e3e771f1439b8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 16 Apr 2021 22:01:05 +0200
Subject: Fix reports of already suspended accounts being recorded (#16047)

---
 app/lib/activitypub/activity/flag.rb | 2 ++
 app/services/report_service.rb       | 2 ++
 2 files changed, 4 insertions(+)

(limited to 'app/services')

diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb
index 8dfc76f0a..b0443849a 100644
--- a/app/lib/activitypub/activity/flag.rb
+++ b/app/lib/activitypub/activity/flag.rb
@@ -10,6 +10,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
     target_accounts.each do |target_account|
       target_statuses = target_statuses_by_account[target_account.id]
 
+      next if target_account.suspended?
+
       ReportService.new.call(
         @account,
         target_account,
diff --git a/app/services/report_service.rb b/app/services/report_service.rb
index 9d9c7d6c9..bc0a8b464 100644
--- a/app/services/report_service.rb
+++ b/app/services/report_service.rb
@@ -10,6 +10,8 @@ class ReportService < BaseService
     @comment        = options.delete(:comment) || ''
     @options        = options
 
+    raise ActiveRecord::RecordNotFound if @target_account.suspended?
+
     create_report!
     notify_staff!
     forward_to_origin! if !@target_account.local? && ActiveModel::Type::Boolean.new.cast(@options[:forward])
-- 
cgit 


From 6d6000f61f7f611358a45efb3c557eb1e0a7e005 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 17 Apr 2021 14:55:46 +0200
Subject: Fix remote reporters not receiving suspend/unsuspend activities
 (#16050)

---
 app/lib/account_reach_finder.rb           | 25 +++++++++++++++++++++++++
 app/services/suspend_account_service.rb   | 12 +++++++++++-
 app/services/unsuspend_account_service.rb | 15 +++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 app/lib/account_reach_finder.rb

(limited to 'app/services')

diff --git a/app/lib/account_reach_finder.rb b/app/lib/account_reach_finder.rb
new file mode 100644
index 000000000..706ce8c1f
--- /dev/null
+++ b/app/lib/account_reach_finder.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class AccountReachFinder
+  def initialize(account)
+    @account = account
+  end
+
+  def inboxes
+    (followers_inboxes + reporters_inboxes + relay_inboxes).uniq
+  end
+
+  private
+
+  def followers_inboxes
+    @account.followers.inboxes
+  end
+
+  def reporters_inboxes
+    Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
+  end
+
+  def relay_inboxes
+    Relay.enabled.pluck(:inbox_url)
+  end
+end
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 9f4da91d4..b8dc8d5e0 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -42,7 +42,13 @@ class SuspendAccountService < BaseService
   end
 
   def distribute_update_actor!
-    ActivityPub::UpdateDistributionWorker.perform_async(@account.id) if @account.local?
+    return unless @account.local?
+
+    account_reach_finder = AccountReachFinder.new(@account)
+
+    ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url|
+      [signed_activity_json, @account.id, inbox_url]
+    end
   end
 
   def unmerge_from_home_timelines!
@@ -90,4 +96,8 @@ class SuspendAccountService < BaseService
       end
     end
   end
+
+  def signed_activity_json
+    @signed_activity_json ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account))
+  end
 end
diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb
index ce9ee48ed..949c670aa 100644
--- a/app/services/unsuspend_account_service.rb
+++ b/app/services/unsuspend_account_service.rb
@@ -12,6 +12,7 @@ class UnsuspendAccountService < BaseService
     merge_into_home_timelines!
     merge_into_list_timelines!
     publish_media_attachments!
+    distribute_update_actor!
   end
 
   private
@@ -36,6 +37,16 @@ class UnsuspendAccountService < BaseService
     # @account would now be nil.
   end
 
+  def distribute_update_actor!
+    return unless @account.local?
+
+    account_reach_finder = AccountReachFinder.new(@account)
+
+    ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url|
+      [signed_activity_json, @account.id, inbox_url]
+    end
+  end
+
   def merge_into_home_timelines!
     @account.followers_for_local_distribution.find_each do |follower|
       FeedManager.instance.merge_into_home(@account, follower)
@@ -81,4 +92,8 @@ class UnsuspendAccountService < BaseService
       end
     end
   end
+
+  def signed_activity_json
+    @signed_activity_json ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account))
+  end
 end
-- 
cgit 


From ca3bc1b09f344f38164aa65d2554cf50d5c10cc0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 17 Apr 2021 15:41:57 +0200
Subject: Refactor StatusReachFinder to handle followers and relays as well
 (#16051)

---
 app/lib/status_reach_finder.rb        | 25 ++++++++++++++++++++++++-
 app/services/remove_status_service.rb | 34 +++++-----------------------------
 2 files changed, 29 insertions(+), 30 deletions(-)

(limited to 'app/services')

diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb
index 35b191dad..3aab3bde0 100644
--- a/app/lib/status_reach_finder.rb
+++ b/app/lib/status_reach_finder.rb
@@ -6,11 +6,22 @@ class StatusReachFinder
   end
 
   def inboxes
-    Account.where(id: reached_account_ids).inboxes
+    (reached_account_inboxes + followers_inboxes + relay_inboxes).uniq
   end
 
   private
 
+  def reached_account_inboxes
+    # When the status is a reblog, there are no interactions with it
+    # directly, we assume all interactions are with the original one
+
+    if @status.reblog?
+      []
+    else
+      Account.where(id: reached_account_ids).inboxes
+    end
+  end
+
   def reached_account_ids
     [
       replied_to_account_id,
@@ -49,4 +60,16 @@ class StatusReachFinder
   def replies_account_ids
     @status.replies.pluck(:account_id)
   end
+
+  def followers_inboxes
+    @status.account.followers.inboxes
+  end
+
+  def relay_inboxes
+    if @status.public_visibility?
+      Relay.enabled.pluck(:inbox_url)
+    else
+      []
+    end
+  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index d642beeaa..2c2a26641 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -27,10 +27,7 @@ class RemoveStatusService < BaseService
         # original object being removed implicitly removes reblogs
         # of it. The Delete activity of the original is forwarded
         # separately.
-        if @account.local? && !@options[:original_removed]
-          remove_from_remote_followers
-          remove_from_remote_reach
-        end
+        remove_from_remote_reach if @account.local? && !@options[:original_removed]
 
         # Since reblogs don't mention anyone, don't get reblogged,
         # favourited and don't contain their own media attachments
@@ -82,13 +79,10 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_remote_reach
-    return if @status.reblog?
-
-    # People who got mentioned in the status, or who
-    # reblogged it from someone else might not follow
-    # the author and wouldn't normally receive the
-    # delete notification - so here, we explicitly
-    # send it to them
+    # Followers, relays, people who got mentioned in the status,
+    # or who reblogged it from someone else might not follow
+    # the author and wouldn't normally receive the delete
+    # notification - so here, we explicitly send it to them
 
     status_reach_finder = StatusReachFinder.new(@status)
 
@@ -97,24 +91,6 @@ class RemoveStatusService < BaseService
     end
   end
 
-  def remove_from_remote_followers
-    ActivityPub::DeliveryWorker.push_bulk(@account.followers.inboxes) do |inbox_url|
-      [signed_activity_json, @account.id, inbox_url]
-    end
-
-    relay! if relayable?
-  end
-
-  def relayable?
-    @status.public_visibility?
-  end
-
-  def relay!
-    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
-      [signed_activity_json, @account.id, inbox_url]
-    end
-  end
-
   def signed_activity_json
     @signed_activity_json ||= Oj.dump(serialize_payload(@status, @status.reblog? ? ActivityPub::UndoAnnounceSerializer : ActivityPub::DeleteSerializer, signer: @account))
   end
-- 
cgit 


From 0b36e3419d4c4ce175f9db266ef5b3a49a9b3974 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 21 Apr 2021 04:46:09 +0200
Subject: Fix processing of remote Delete activities (#16084)

* Add tests

* Ensure deleted statuses are marked as such

* Save some redis memory by not storing URIs in delete_upon_arrival values

* Avoid possible race condition when processing incoming Deletes

* Avoid potential duplicate Delete forwards

* Lower lock durations to reduce issues in case of hard crash of the Rails process

* Check for `lock.aquired?` and improve comment

* Refactor RedisLock usage in app/lib/activitypub

* Fix using incorrect or non-existent sender for relaying Deletes
---
 app/lib/activitypub/activity.rb              | 14 ++++++-
 app/lib/activitypub/activity/announce.rb     | 36 +++++++---------
 app/lib/activitypub/activity/create.rb       | 36 +++++-----------
 app/lib/activitypub/activity/delete.rb       | 62 +++++++++++++++-------------
 app/services/remove_status_service.rb        |  2 +
 spec/lib/activitypub/activity/delete_spec.rb | 20 +++++++++
 6 files changed, 91 insertions(+), 79 deletions(-)

(limited to 'app/services')

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 2b5d3ffc2..3baee4ca4 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -144,7 +144,7 @@ class ActivityPub::Activity
   end
 
   def delete_later!(uri)
-    redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri)
+    redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, true)
   end
 
   def status_from_object
@@ -210,12 +210,22 @@ class ActivityPub::Activity
     end
   end
 
-  def lock_or_return(key, expire_after = 7.days.seconds)
+  def lock_or_return(key, expire_after = 2.hours.seconds)
     yield if redis.set(key, true, nx: true, ex: expire_after)
   ensure
     redis.del(key)
   end
 
+  def lock_or_fail(key)
+    RedisLock.acquire({ redis: Redis.current, key: key }) do |lock|
+      if lock.acquired?
+        yield
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
+  end
+
   def fetch?
     !@options[:delivery]
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index ae8b2db75..a1081522e 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -4,29 +4,25 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   def perform
     return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity?
 
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        original_status = status_from_object
+    lock_or_fail("announce:#{@object['id']}") do
+      original_status = status_from_object
 
-        return reject_payload! if original_status.nil? || !announceable?(original_status)
+      return reject_payload! if original_status.nil? || !announceable?(original_status)
 
-        @status = Status.find_by(account: @account, reblog: original_status)
+      @status = Status.find_by(account: @account, reblog: original_status)
 
-        return @status unless @status.nil?
+      return @status unless @status.nil?
 
-        @status = Status.create!(
-          account: @account,
-          reblog: original_status,
-          uri: @json['id'],
-          created_at: @json['published'],
-          override_timestamps: @options[:override_timestamps],
-          visibility: visibility_from_audience
-        )
+      @status = Status.create!(
+        account: @account,
+        reblog: original_status,
+        uri: @json['id'],
+        created_at: @json['published'],
+        override_timestamps: @options[:override_timestamps],
+        visibility: visibility_from_audience
+      )
 
-        distribute(@status)
-      else
-        raise Mastodon::RaceConditionError
-      end
+      distribute(@status)
     end
 
     @status
@@ -69,8 +65,4 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   def reblog_of_local_status?
     status_from_uri(object_uri)&.account&.local?
   end
-
-  def lock_options
-    { redis: Redis.current, key: "announce:#{@object['id']}" }
-  end
 end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 9f6dd9ce0..c7a655d9d 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -45,19 +45,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def create_status
     return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity?
 
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator
+    lock_or_fail("create:#{object_uri}") do
+      return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator
 
-        @status = find_existing_status
+      @status = find_existing_status
 
-        if @status.nil?
-          process_status
-        elsif @options[:delivered_to_account_id].present?
-          postprocess_audience_and_deliver
-        end
-      else
-        raise Mastodon::RaceConditionError
+      if @status.nil?
+        process_status
+      elsif @options[:delivered_to_account_id].present?
+        postprocess_audience_and_deliver
       end
     end
 
@@ -313,13 +309,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     poll = replied_to_status.preloadable_poll
     already_voted = true
 
-    RedisLock.acquire(poll_lock_options) do |lock|
-      if lock.acquired?
-        already_voted = poll.votes.where(account: @account).exists?
-        poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
-      else
-        raise Mastodon::RaceConditionError
-      end
+    lock_or_fail("vote:#{replied_to_status.poll_id}:#{@account.id}") do
+      already_voted = poll.votes.where(account: @account).exists?
+      poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
     end
 
     increment_voters_count! unless already_voted
@@ -508,12 +500,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     poll.reload
     retry
   end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{object_uri}" }
-  end
-
-  def poll_lock_options
-    { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
-  end
 end
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 2e5293b83..801647cf7 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -20,33 +20,35 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   def delete_note
     return if object_uri.nil?
 
-    unless invalid_origin?(object_uri)
-      RedisLock.acquire(lock_options) { |_lock| delete_later!(object_uri) }
-      Tombstone.find_or_create_by(uri: object_uri, account: @account)
-    end
+    lock_or_return("delete_status_in_progress:#{object_uri}", 5.minutes.seconds) do
+      unless invalid_origin?(object_uri)
+        # This lock ensures a concurrent `ActivityPub::Activity::Create` either
+        # does not create a status at all, or has finished saving it to the
+        # database before we try to load it.
+        # Without the lock, `delete_later!` could be called after `delete_arrived_first?`
+        # and `Status.find` before `Status.create!`
+        lock_or_fail("create:#{object_uri}") { delete_later!(object_uri) }
 
-    @status   = Status.find_by(uri: object_uri, account: @account)
-    @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
+        Tombstone.find_or_create_by(uri: object_uri, account: @account)
+      end
 
-    return if @status.nil?
+      @status   = Status.find_by(uri: object_uri, account: @account)
+      @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
 
-    if @status.distributable?
-      forward_for_reply
-      forward_for_reblogs
-    end
+      return if @status.nil?
 
-    delete_now!
+      forward! if @json['signature'].present? && @status.distributable?
+      delete_now!
+    end
   end
 
-  def forward_for_reblogs
-    return if @json['signature'].blank?
-
-    rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)
-    inboxes        = Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes - [@account.preferred_inbox_url]
+  def rebloggers_ids
+    return @rebloggers_ids if defined?(@rebloggers_ids)
+    @rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)
+  end
 
-    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, rebloggers_ids.first, inbox_url]
-    end
+  def inboxes_for_reblogs
+    Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes
   end
 
   def replied_to_status
@@ -58,13 +60,19 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
     !replied_to_status.nil? && replied_to_status.account.local?
   end
 
-  def forward_for_reply
-    return unless @json['signature'].present? && reply_to_local?
+  def inboxes_for_reply
+    replied_to_status.account.followers.inboxes
+  end
+
+  def forward!
+    inboxes = inboxes_for_reblogs
+    inboxes += inboxes_for_reply if reply_to_local?
+    inboxes -= [@account.preferred_inbox_url]
 
-    inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url]
+    sender_id = reply_to_local? ? replied_to_status.account_id : rebloggers_ids.first
 
-    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, replied_to_status.account_id, inbox_url]
+    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes.uniq) do |inbox_url|
+      [payload, sender_id, inbox_url]
     end
   end
 
@@ -75,8 +83,4 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   def payload
     @payload ||= Oj.dump(@json)
   end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{object_uri}" }
-  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 2c2a26641..6e4d6e72a 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -16,6 +16,8 @@ class RemoveStatusService < BaseService
     @account  = status.account
     @options  = options
 
+    @status.discard
+
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         remove_from_self if @account.local?
diff --git a/spec/lib/activitypub/activity/delete_spec.rb b/spec/lib/activitypub/activity/delete_spec.rb
index 37b93ecf7..9dfb8a61b 100644
--- a/spec/lib/activitypub/activity/delete_spec.rb
+++ b/spec/lib/activitypub/activity/delete_spec.rb
@@ -49,4 +49,24 @@ RSpec.describe ActivityPub::Activity::Delete do
       end
     end
   end
+
+  context 'when the status has been reported' do
+    describe '#perform' do
+      subject { described_class.new(json, sender) }
+      let!(:reporter) { Fabricate(:account) }
+
+      before do
+        reporter.reports.create!(target_account: status.account, status_ids: [status.id], forwarded: false)
+        subject.perform
+      end
+
+      it 'marks the status as deleted' do
+        expect(Status.find_by(id: status.id)).to be_nil
+      end
+
+      it 'actually keeps a copy for inspection' do
+        expect(Status.with_discarded.find_by(id: status.id)).to_not be_nil
+      end
+    end
+  end
 end
-- 
cgit 


From daccc07dc170627b17564402296f6c8631d0cd97 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 24 Apr 2021 17:01:43 +0200
Subject: Change auto-following admin-selected accounts, show in
 recommendations (#16078)

---
 app/controllers/api/v1/suggestions_controller.rb   | 10 ++--
 app/lib/potential_friendship_tracker.rb            | 10 ----
 app/models/account_suggestions.rb                  | 25 +++++---
 app/models/account_suggestions/global_source.rb    | 37 ++++++++++++
 .../past_interactions_source.rb                    | 36 ++++++++++++
 app/models/account_suggestions/setting_source.rb   | 68 ++++++++++++++++++++++
 app/models/account_suggestions/source.rb           | 34 +++++++++++
 app/models/account_suggestions/suggestion.rb       |  7 +++
 app/models/follow_recommendation.rb                | 15 -----
 app/models/form/admin_settings.rb                  |  2 -
 app/services/bootstrap_timeline_service.rb         | 37 +-----------
 app/validators/existing_username_validator.rb      | 24 ++++++--
 app/views/admin/settings/edit.html.haml            |  5 +-
 config/locales/en.yml                              |  7 +--
 config/settings.yml                                |  1 -
 spec/services/bootstrap_timeline_service_spec.rb   | 38 ------------
 16 files changed, 228 insertions(+), 128 deletions(-)
 create mode 100644 app/models/account_suggestions/global_source.rb
 create mode 100644 app/models/account_suggestions/past_interactions_source.rb
 create mode 100644 app/models/account_suggestions/setting_source.rb
 create mode 100644 app/models/account_suggestions/source.rb
 create mode 100644 app/models/account_suggestions/suggestion.rb

(limited to 'app/services')

diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb
index b2788cc76..9737ae5cb 100644
--- a/app/controllers/api/v1/suggestions_controller.rb
+++ b/app/controllers/api/v1/suggestions_controller.rb
@@ -5,20 +5,20 @@ class Api::V1::SuggestionsController < Api::BaseController
 
   before_action -> { doorkeeper_authorize! :read }
   before_action :require_user!
-  before_action :set_accounts
 
   def index
-    render json: @accounts, each_serializer: REST::AccountSerializer
+    suggestions = suggestions_source.get(current_account, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT))
+    render json: suggestions.map(&:account), each_serializer: REST::AccountSerializer
   end
 
   def destroy
-    PotentialFriendshipTracker.remove(current_account.id, params[:id])
+    suggestions_source.remove(current_account, params[:id])
     render_empty
   end
 
   private
 
-  def set_accounts
-    @accounts = PotentialFriendshipTracker.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT))
+  def suggestions_source
+    AccountSuggestions::PastInteractionsSource.new
   end
 end
diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb
index e72d454b6..f5bc20346 100644
--- a/app/lib/potential_friendship_tracker.rb
+++ b/app/lib/potential_friendship_tracker.rb
@@ -27,15 +27,5 @@ class PotentialFriendshipTracker
     def remove(account_id, target_account_id)
       redis.zrem("interactions:#{account_id}", target_account_id)
     end
-
-    def get(account, limit)
-      account_ids = redis.zrevrange("interactions:#{account.id}", 0, limit)
-
-      return [] if account_ids.empty? || limit < 1
-
-      accounts = Account.searchable.where(id: account_ids).index_by(&:id)
-
-      account_ids.map { |id| accounts[id.to_i] }.compact
-    end
   end
 end
diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb
index 7fe9d618e..d1774e62f 100644
--- a/app/models/account_suggestions.rb
+++ b/app/models/account_suggestions.rb
@@ -1,17 +1,28 @@
 # frozen_string_literal: true
 
 class AccountSuggestions
-  class Suggestion < ActiveModelSerializers::Model
-    attributes :account, :source
-  end
+  SOURCES = [
+    AccountSuggestions::SettingSource,
+    AccountSuggestions::PastInteractionsSource,
+    AccountSuggestions::GlobalSource,
+  ].freeze
 
   def self.get(account, limit)
-    suggestions = PotentialFriendshipTracker.get(account, limit).map { |target_account| Suggestion.new(account: target_account, source: :past_interaction) }
-    suggestions.concat(FollowRecommendation.get(account, limit - suggestions.size, suggestions.map { |suggestion| suggestion.account.id }).map { |target_account| Suggestion.new(account: target_account, source: :global) }) if suggestions.size < limit
-    suggestions
+    SOURCES.each_with_object([]) do |source_class, suggestions|
+      source_suggestions = source_class.new.get(
+        account,
+        skip_account_ids: suggestions.map(&:account_id),
+        limit: limit - suggestions.size
+      )
+
+      suggestions.concat(source_suggestions)
+    end
   end
 
   def self.remove(account, target_account_id)
-    PotentialFriendshipTracker.remove(account.id, target_account_id)
+    SOURCES.each do |source_class|
+      source = source_class.new
+      source.remove(account, target_account_id)
+    end
   end
 end
diff --git a/app/models/account_suggestions/global_source.rb b/app/models/account_suggestions/global_source.rb
new file mode 100644
index 000000000..ac764de50
--- /dev/null
+++ b/app/models/account_suggestions/global_source.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::GlobalSource < AccountSuggestions::Source
+  def key
+    :global
+  end
+
+  def get(account, skip_account_ids: [], limit: 40)
+    account_ids = account_ids_for_locale(account.user_locale) - [account.id] - skip_account_ids
+
+    as_ordered_suggestions(
+      scope(account).where(id: account_ids),
+      account_ids
+    ).take(limit)
+  end
+
+  def remove(_account, _target_account_id)
+    nil
+  end
+
+  private
+
+  def scope(account)
+    Account.searchable
+           .followable_by(account)
+           .not_excluded_by_account(account)
+           .not_domain_blocked_by_account(account)
+  end
+
+  def account_ids_for_locale(locale)
+    Redis.current.zrevrange("follow_recommendations:#{locale}", 0, -1).map(&:to_i)
+  end
+
+  def to_ordered_list_key(account)
+    account.id
+  end
+end
diff --git a/app/models/account_suggestions/past_interactions_source.rb b/app/models/account_suggestions/past_interactions_source.rb
new file mode 100644
index 000000000..d169394f1
--- /dev/null
+++ b/app/models/account_suggestions/past_interactions_source.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::PastInteractionsSource < AccountSuggestions::Source
+  include Redisable
+
+  def key
+    :past_interactions
+  end
+
+  def get(account, skip_account_ids: [], limit: 40)
+    account_ids = account_ids_for_account(account.id, limit + skip_account_ids.size) - skip_account_ids
+
+    as_ordered_suggestions(
+      scope.where(id: account_ids),
+      account_ids
+    ).take(limit)
+  end
+
+  def remove(account, target_account_id)
+    redis.zrem("interactions:#{account.id}", target_account_id)
+  end
+
+  private
+
+  def scope
+    Account.searchable
+  end
+
+  def account_ids_for_account(account_id, limit)
+    redis.zrevrange("interactions:#{account_id}", 0, limit).map(&:to_i)
+  end
+
+  def to_ordered_list_key(account)
+    account.id
+  end
+end
diff --git a/app/models/account_suggestions/setting_source.rb b/app/models/account_suggestions/setting_source.rb
new file mode 100644
index 000000000..be9eff233
--- /dev/null
+++ b/app/models/account_suggestions/setting_source.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::SettingSource < AccountSuggestions::Source
+  def key
+    :staff
+  end
+
+  def get(account, skip_account_ids: [], limit: 40)
+    return [] unless setting_enabled?
+
+    as_ordered_suggestions(
+      scope(account).where(setting_to_where_condition).where.not(id: skip_account_ids),
+      usernames_and_domains
+    ).take(limit)
+  end
+
+  def remove(_account, _target_account_id)
+    nil
+  end
+
+  private
+
+  def scope(account)
+    Account.searchable
+           .followable_by(account)
+           .not_excluded_by_account(account)
+           .not_domain_blocked_by_account(account)
+           .where(locked: false)
+           .where.not(id: account.id)
+  end
+
+  def usernames_and_domains
+    @usernames_and_domains ||= setting_to_usernames_and_domains
+  end
+
+  def setting_enabled?
+    setting.present?
+  end
+
+  def setting_to_where_condition
+    usernames_and_domains.map do |(username, domain)|
+      Arel::Nodes::Grouping.new(
+        Account.arel_table[:username].lower.eq(username.downcase).and(
+          Account.arel_table[:domain].lower.eq(domain&.downcase)
+        )
+      )
+    end.reduce(:or)
+  end
+
+  def setting_to_usernames_and_domains
+    setting.split(',').map do |str|
+      username, domain = str.strip.gsub(/\A@/, '').split('@', 2)
+      domain           = nil if TagManager.instance.local_domain?(domain)
+
+      next if username.blank?
+
+      [username, domain]
+    end.compact
+  end
+
+  def setting
+    Setting.bootstrap_timeline_accounts
+  end
+
+  def to_ordered_list_key(account)
+    [account.username, account.domain]
+  end
+end
diff --git a/app/models/account_suggestions/source.rb b/app/models/account_suggestions/source.rb
new file mode 100644
index 000000000..bd1068d20
--- /dev/null
+++ b/app/models/account_suggestions/source.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::Source
+  def key
+    raise NotImplementedError
+  end
+
+  def get(_account, **kwargs)
+    raise NotImplementedError
+  end
+
+  def remove(_account, target_account_id)
+    raise NotImplementedError
+  end
+
+  protected
+
+  def as_ordered_suggestions(scope, ordered_list)
+    return [] if ordered_list.empty?
+
+    map = scope.index_by(&method(:to_ordered_list_key))
+
+    ordered_list.map { |ordered_list_key| map[ordered_list_key] }.compact.map do |account|
+      AccountSuggestions::Suggestion.new(
+        account: account,
+        source: key
+      )
+    end
+  end
+
+  def to_ordered_list_key(_account)
+    raise NotImplementedError
+  end
+end
diff --git a/app/models/account_suggestions/suggestion.rb b/app/models/account_suggestions/suggestion.rb
new file mode 100644
index 000000000..2c6f4d27f
--- /dev/null
+++ b/app/models/account_suggestions/suggestion.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::Suggestion < ActiveModelSerializers::Model
+  attributes :account, :source
+
+  delegate :id, to: :account, prefix: true
+end
diff --git a/app/models/follow_recommendation.rb b/app/models/follow_recommendation.rb
index c4355224d..6670b6560 100644
--- a/app/models/follow_recommendation.rb
+++ b/app/models/follow_recommendation.rb
@@ -21,19 +21,4 @@ class FollowRecommendation < ApplicationRecord
   def readonly?
     true
   end
-
-  def self.get(account, limit, exclude_account_ids = [])
-    account_ids = Redis.current.zrevrange("follow_recommendations:#{account.user_locale}", 0, -1).map(&:to_i) - exclude_account_ids - [account.id]
-
-    return [] if account_ids.empty? || limit < 1
-
-    accounts = Account.followable_by(account)
-                      .not_excluded_by_account(account)
-                      .not_domain_blocked_by_account(account)
-                      .where(id: account_ids)
-                      .limit(limit)
-                      .index_by(&:id)
-
-    account_ids.map { |id| accounts[id] }.compact
-  end
 end
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index b5c3dcdbe..6fc7c56fd 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -16,7 +16,6 @@ class Form::AdminSettings
     open_deletion
     timeline_preview
     show_staff_badge
-    enable_bootstrap_timeline_accounts
     bootstrap_timeline_accounts
     theme
     min_invite_role
@@ -41,7 +40,6 @@ class Form::AdminSettings
     open_deletion
     timeline_preview
     show_staff_badge
-    enable_bootstrap_timeline_accounts
     activity_api_enabled
     peers_api_enabled
     show_known_fediverse_at_about_page
diff --git a/app/services/bootstrap_timeline_service.rb b/app/services/bootstrap_timeline_service.rb
index 8412aa7e7..e1a1b98c3 100644
--- a/app/services/bootstrap_timeline_service.rb
+++ b/app/services/bootstrap_timeline_service.rb
@@ -5,48 +5,13 @@ class BootstrapTimelineService < BaseService
     @source_account = source_account
 
     autofollow_inviter!
-    autofollow_bootstrap_timeline_accounts! if Setting.enable_bootstrap_timeline_accounts
   end
 
   private
 
   def autofollow_inviter!
     return unless @source_account&.user&.invite&.autofollow?
-    FollowService.new.call(@source_account, @source_account.user.invite.user.account)
-  end
-
-  def autofollow_bootstrap_timeline_accounts!
-    bootstrap_timeline_accounts.each do |target_account|
-      begin
-        FollowService.new.call(@source_account, target_account)
-      rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
-        nil
-      end
-    end
-  end
-
-  def bootstrap_timeline_accounts
-    return @bootstrap_timeline_accounts if defined?(@bootstrap_timeline_accounts)
-
-    @bootstrap_timeline_accounts = bootstrap_timeline_accounts_usernames.empty? ? admin_accounts : local_unlocked_accounts(bootstrap_timeline_accounts_usernames)
-  end
-
-  def bootstrap_timeline_accounts_usernames
-    @bootstrap_timeline_accounts_usernames ||= (Setting.bootstrap_timeline_accounts || '').split(',').map { |str| str.strip.gsub(/\A@/, '') }.reject(&:blank?)
-  end
 
-  def admin_accounts
-    User.admins
-        .includes(:account)
-        .where(accounts: { locked: false })
-        .map(&:account)
-  end
-
-  def local_unlocked_accounts(usernames)
-    Account.local
-           .without_suspended
-           .where(username: usernames)
-           .where(locked: false)
-           .where(moved_to_account_id: nil)
+    FollowService.new.call(@source_account, @source_account.user.invite.user.account)
   end
 end
diff --git a/app/validators/existing_username_validator.rb b/app/validators/existing_username_validator.rb
index 723302ec9..afbe0c635 100644
--- a/app/validators/existing_username_validator.rb
+++ b/app/validators/existing_username_validator.rb
@@ -4,11 +4,25 @@ class ExistingUsernameValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
     return if value.blank?
 
-    if options[:multiple]
-      missing_usernames = value.split(',').map { |username| username.strip.gsub(/\A@/, '') }.filter_map { |username| username unless Account.find_local(username) }
-      record.errors.add(attribute, I18n.t('existing_username_validator.not_found_multiple', usernames: missing_usernames.join(', '))) if missing_usernames.any?
-    else
-      record.errors.add(attribute, I18n.t('existing_username_validator.not_found')) unless Account.find_local(value.strip.gsub(/\A@/, ''))
+    usernames_and_domains = begin
+      value.split(',').map do |str|
+        username, domain = str.strip.gsub(/\A@/, '').split('@')
+        domain = nil if TagManager.instance.local_domain?(domain)
+
+        next if username.blank?
+
+        [str, username, domain]
+      end.compact
+    end
+
+    usernames_with_no_accounts = usernames_and_domains.filter_map do |(str, username, domain)|
+      str unless Account.find_remote(username, domain)
+    end
+
+    if usernames_with_no_accounts.any? && options[:multiple]
+      record.errors.add(attribute, I18n.t('existing_username_validator.not_found_multiple', usernames: usernames_with_no_accounts.join(', ')))
+    elsif usernames_with_no_accounts.any? || usernames_and_domains.size > 1
+      record.errors.add(attribute, I18n.t('existing_username_validator.not_found'))
     end
   end
 end
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 7783dbfeb..33bfc43d3 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -50,10 +50,7 @@
   %hr.spacer/
 
   .fields-group
-    = f.input :enable_bootstrap_timeline_accounts, as: :boolean, wrapper: :with_label, label: t('admin.settings.enable_bootstrap_timeline_accounts.title'), hint: t('admin.settings.enable_bootstrap_timeline_accounts.desc_html')
-
-  .fields-group
-    = f.input :bootstrap_timeline_accounts, wrapper: :with_block_label, label: t('admin.settings.bootstrap_timeline_accounts.title'), hint: t('admin.settings.bootstrap_timeline_accounts.desc_html'), disabled: !Setting.enable_bootstrap_timeline_accounts
+    = f.input :bootstrap_timeline_accounts, wrapper: :with_block_label, label: t('admin.settings.bootstrap_timeline_accounts.title'), hint: t('admin.settings.bootstrap_timeline_accounts.desc_html')
 
   %hr.spacer/
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 765f71250..1b41ee063 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -564,8 +564,8 @@ en:
         desc_html: Counts of locally published posts, active users, and new registrations in weekly buckets
         title: Publish aggregate statistics about user activity in the API
       bootstrap_timeline_accounts:
-        desc_html: Separate multiple usernames by comma. Only local and unlocked accounts will work. Default when empty is all local admins.
-        title: Default follows for new users
+        desc_html: Separate multiple usernames by comma. These accounts will be guaranteed to be shown in follow recommendations
+        title: Recommend these accounts to new users
       contact_information:
         email: Business e-mail
         username: Contact username
@@ -582,9 +582,6 @@ en:
         users: To logged-in local users
       domain_blocks_rationale:
         title: Show rationale
-      enable_bootstrap_timeline_accounts:
-        desc_html: Make new users automatically follow configured accounts so their home feed doesn't start out empty
-        title: Enable default follows for new users
       hero:
         desc_html: Displayed on the frontpage. At least 600x100px recommended. When not set, falls back to server thumbnail
         title: Hero image
diff --git a/config/settings.yml b/config/settings.yml
index b79ea620c..06cee2532 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -62,7 +62,6 @@ defaults: &defaults
     - mod
     - moderator
   disallowed_hashtags: # space separated string or list of hashtags without the hash
-  enable_bootstrap_timeline_accounts: true
   bootstrap_timeline_accounts: ''
   activity_api_enabled: true
   peers_api_enabled: true
diff --git a/spec/services/bootstrap_timeline_service_spec.rb b/spec/services/bootstrap_timeline_service_spec.rb
index a28d2407c..880ca4f0d 100644
--- a/spec/services/bootstrap_timeline_service_spec.rb
+++ b/spec/services/bootstrap_timeline_service_spec.rb
@@ -1,42 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe BootstrapTimelineService, type: :service do
-  subject { described_class.new }
-
-  describe '#call' do
-    let(:source_account) { Fabricate(:account) }
-
-    context 'when setting is empty' do
-      let!(:admin) { Fabricate(:user, admin: true) }
-
-      before do
-        Setting.bootstrap_timeline_accounts = nil
-        subject.call(source_account)
-      end
-
-      it 'follows admin accounts from account' do
-        expect(source_account.following?(admin.account)).to be true
-      end
-    end
-
-    context 'when setting is set' do
-      let!(:alice) { Fabricate(:account, username: 'alice') }
-      let!(:bob)   { Fabricate(:account, username: 'bob') }
-      let!(:eve)   { Fabricate(:account, username: 'eve', suspended: true) }
-
-      before do
-        Setting.bootstrap_timeline_accounts = 'alice, @bob, eve, unknown'
-        subject.call(source_account)
-      end
-
-      it 'follows found accounts from account' do
-        expect(source_account.following?(alice)).to be true
-        expect(source_account.following?(bob)).to be true
-      end
-
-      it 'does not follow suspended account' do
-        expect(source_account.following?(eve)).to be false
-      end
-    end
-  end
 end
-- 
cgit 


From f627d2eb938d220eb767b0211b66b4281c921f75 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 1 May 2021 23:18:59 +0200
Subject: Fix trying to fetch key from empty URI when verifying HTTP signature
 (#16100)

---
 app/helpers/jsonld_helper.rb                         | 2 +-
 app/services/activitypub/fetch_remote_key_service.rb | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 1c473efa3..62eb50f78 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -67,7 +67,7 @@ module JsonLdHelper
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)
 
-      return unless json
+      return if !json.is_a?(Hash) || unsupported_uri_scheme?(json['id'])
 
       uri = json['id']
     end
diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb
index df17d9079..c48288b3b 100644
--- a/app/services/activitypub/fetch_remote_key_service.rb
+++ b/app/services/activitypub/fetch_remote_key_service.rb
@@ -5,6 +5,8 @@ class ActivityPub::FetchRemoteKeyService < BaseService
 
   # Returns account that owns the key
   def call(uri, id: true, prefetched_body: nil)
+    return if uri.blank?
+
     if prefetched_body.nil?
       if id
         @json = fetch_resource_without_id_validation(uri)
-- 
cgit 


From fab65848d2eb8065ef3e49aaca4e4fb33f94f2b1 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 May 2021 04:45:08 +0200
Subject: Fix empty home feed before first follow has finished processing
 (#16152)

Change queue of merge worker from pull to default
---
 app/models/concerns/account_interactions.rb | 8 ++++++++
 app/models/user.rb                          | 4 +---
 app/services/follow_service.rb              | 9 +++++++++
 app/workers/merge_worker.rb                 | 4 ++--
 4 files changed, 20 insertions(+), 5 deletions(-)

(limited to 'app/services')

diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index 51e8e04a8..958f6c78e 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -184,6 +184,14 @@ module AccountInteractions
     active_relationships.where(target_account: other_account).exists?
   end
 
+  def following_anyone?
+    active_relationships.exists?
+  end
+
+  def not_following_anyone?
+    !following_anyone?
+  end
+
   def blocking?(other_account)
     block_relationships.where(target_account: other_account).exists?
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 5a149f573..0440627c5 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -458,9 +458,7 @@ class User < ApplicationRecord
   end
 
   def regenerate_feed!
-    return unless Redis.current.setnx("account:#{account_id}:regeneration", true)
-    Redis.current.expire("account:#{account_id}:regeneration", 1.day.seconds)
-    RegenerationWorker.perform_async(account_id)
+    RegenerationWorker.perform_async(account_id) if Redis.current.set("account:#{account_id}:regeneration", true, nx: true, ex: 1.day.seconds)
   end
 
   def needs_feed_update?
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index d3db07a74..329262cca 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -30,6 +30,11 @@ class FollowService < BaseService
 
     ActivityTracker.increment('activity:interactions')
 
+    # When an account follows someone for the first time, avoid showing
+    # an empty home feed while the follow request is being processed
+    # and the feeds are being merged
+    mark_home_feed_as_partial! if @source_account.not_following_anyone?
+
     if (@target_account.locked? && !@options[:bypass_locked]) || @source_account.silenced? || @target_account.activitypub?
       request_follow!
     elsif @target_account.local?
@@ -39,6 +44,10 @@ class FollowService < BaseService
 
   private
 
+  def mark_home_feed_as_partial!
+    redis.set("account:#{@source_account.id}:regeneration", true, nx: true, ex: 1.day.seconds)
+  end
+
   def following_not_possible?
     @target_account.nil? || @target_account.id == @source_account.id || @target_account.suspended?
   end
diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb
index 74ef7d4da..6ebb9a400 100644
--- a/app/workers/merge_worker.rb
+++ b/app/workers/merge_worker.rb
@@ -3,11 +3,11 @@
 class MergeWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull'
-
   def perform(from_account_id, into_account_id)
     FeedManager.instance.merge_into_home(Account.find(from_account_id), Account.find(into_account_id))
   rescue ActiveRecord::RecordNotFound
     true
+  ensure
+    Redis.current.del("account:#{into_account_id}:regeneration")
   end
 end
-- 
cgit 


From 2c77d97e0d59e045a9b04fccc83f0f24d190d8d8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 7 May 2021 14:33:19 +0200
Subject: Add joined date to profiles in web UI (#16169)

---
 .../mastodon/features/account/components/header.js         |  2 ++
 app/javascript/mastodon/locales/defaultMessages.json       | 14 +++++++++-----
 app/javascript/styles/mastodon/components.scss             | 11 +++++++++++
 app/serializers/activitypub/actor_serializer.rb            |  6 +++++-
 app/serializers/rest/account_serializer.rb                 |  4 ++++
 app/services/activitypub/process_account_service.rb        |  3 ++-
 spec/lib/activitypub/activity/update_spec.rb               |  2 +-
 7 files changed, 34 insertions(+), 8 deletions(-)

(limited to 'app/services')

diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js
index 8e49486bd..20641121f 100644
--- a/app/javascript/mastodon/features/account/components/header.js
+++ b/app/javascript/mastodon/features/account/components/header.js
@@ -326,6 +326,8 @@ class Header extends ImmutablePureComponent {
               {account.get('id') !== me && !suspended && <AccountNoteContainer account={account} />}
 
               {account.get('note').length > 0 && account.get('note') !== '<p></p>' && <div className='account__header__content translate' dangerouslySetInnerHTML={content} />}
+
+              <div className='account__header__joined'><FormattedMessage id='account.joined' defaultMessage='Joined {date}' values={{ date: intl.formatDate(account.get('created_at'), { year: 'numeric', month: 'short', day: '2-digit' }) }} /></div>
             </div>
 
             {!suspended && (
diff --git a/app/javascript/mastodon/locales/defaultMessages.json b/app/javascript/mastodon/locales/defaultMessages.json
index 86cf83403..172fae81f 100644
--- a/app/javascript/mastodon/locales/defaultMessages.json
+++ b/app/javascript/mastodon/locales/defaultMessages.json
@@ -909,6 +909,10 @@
       {
         "defaultMessage": "Group",
         "id": "account.badges.group"
+      },
+      {
+        "defaultMessage": "Joined {date}",
+        "id": "account.joined"
       }
     ],
     "path": "app/javascript/mastodon/features/account/components/header.json"
@@ -1919,12 +1923,12 @@
         "id": "home.hide_announcements"
       },
       {
-        "defaultMessage": "Your home timeline is empty! Visit {public} or use search to get started and meet other users.",
+        "defaultMessage": "Your home timeline is empty! Follow more people to fill it up. {suggestions}",
         "id": "empty_column.home"
       },
       {
-        "defaultMessage": "the public timeline",
-        "id": "empty_column.home.public_timeline"
+        "defaultMessage": "See some suggestions",
+        "id": "empty_column.home.suggestions"
       }
     ],
     "path": "app/javascript/mastodon/features/home_timeline/index.json"
@@ -2417,7 +2421,7 @@
         "id": "notifications.mark_as_read"
       },
       {
-        "defaultMessage": "You don't have any notifications yet. Interact with others to start the conversation.",
+        "defaultMessage": "You don't have any notifications yet. When other people interact with you, you will see it here.",
         "id": "empty_column.notifications"
       }
     ],
@@ -3249,4 +3253,4 @@
     ],
     "path": "app/javascript/mastodon/features/video/index.json"
   }
-]
\ No newline at end of file
+]
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 74a181603..d3dd1af60 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -6769,6 +6769,17 @@ noscript {
       }
     }
 
+    .account__header__joined {
+      font-size: 14px;
+      padding: 5px 15px;
+      color: $darker-text-color;
+
+      .columns-area--mobile & {
+        padding-left: 20px;
+        padding-right: 20px;
+      }
+    }
+
     .account__header__fields {
       margin: 0;
       border-top: 1px solid lighten($ui-base-color, 12%);
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 759ef30f9..d92aae7b3 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -13,7 +13,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
              :inbox, :outbox, :featured, :featured_tags,
              :preferred_username, :name, :summary,
              :url, :manually_approves_followers,
-             :discoverable
+             :discoverable, :published
 
   has_one :public_key, serializer: ActivityPub::PublicKeySerializer
 
@@ -158,6 +158,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
     !object.suspended? && !object.also_known_as.empty?
   end
 
+  def published
+    object.created_at.midnight.iso8601
+  end
+
   class CustomEmojiSerializer < ActivityPub::EmojiSerializer
   end
 
diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb
index 189a62d0e..219f8075a 100644
--- a/app/serializers/rest/account_serializer.rb
+++ b/app/serializers/rest/account_serializer.rb
@@ -55,6 +55,10 @@ class REST::AccountSerializer < ActiveModel::Serializer
     full_asset_url(object.suspended? ? object.header.default_url : object.header_static_url)
   end
 
+  def created_at
+    object.created_at.midnight.iso8601
+  end
+
   def last_status_at
     object.last_status_at&.to_date&.iso8601
   end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 6afeb92d6..bb2e8f665 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -87,6 +87,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.url                     = url || @uri
     @account.uri                     = @uri
     @account.actor_type              = actor_type
+    @account.created_at              = @json['published'] if @json['published'].present?
   end
 
   def set_immediate_attributes!
@@ -101,7 +102,7 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def set_fetchable_key!
-    @account.public_key        = public_key || ''
+    @account.public_key = public_key || ''
   end
 
   def set_fetchable_attributes!
diff --git a/spec/lib/activitypub/activity/update_spec.rb b/spec/lib/activitypub/activity/update_spec.rb
index 42da29860..1c9bcf43b 100644
--- a/spec/lib/activitypub/activity/update_spec.rb
+++ b/spec/lib/activitypub/activity/update_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe ActivityPub::Activity::Update do
   end
 
   let(:modified_sender) do
-    sender.dup.tap do |modified_sender|
+    sender.tap do |modified_sender|
       modified_sender.display_name = 'Totally modified now'
     end
   end
-- 
cgit 


From 74081433d0078784b7c2139f6caaa812740632b2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 7 May 2021 14:33:43 +0200
Subject: Change trending hashtags to be affected be reblogs (#16164)

If a status with a hashtag becomes very popular, it stands to
reason that the hashtag should have a chance at trending

Fix no stats being recorded for hashtags that are not allowed
to trend, and stop ignoring bots

Remove references to hashtags in profile directory from the code
and the admin UI
---
 app/controllers/directories_controller.rb          | 10 ------
 app/lib/activitypub/activity/announce.rb           |  4 +++
 app/lib/activitypub/activity/create.rb             |  2 +-
 app/models/account.rb                              | 11 ++-----
 app/models/account_tag_stat.rb                     | 24 --------------
 app/models/tag.rb                                  | 38 +++++-----------------
 app/models/tag_filter.rb                           |  2 --
 app/models/trending_tags.rb                        | 12 ++++---
 app/services/process_hashtags_service.rb           |  3 +-
 app/services/reblog_service.rb                     | 11 +++++++
 app/views/admin/tags/_tag.html.haml                |  2 --
 app/views/admin/tags/index.html.haml               |  9 ++---
 app/views/admin/tags/show.html.haml                | 13 ++------
 config/locales/en.yml                              |  7 ++--
 config/locales/simple_form.en.yml                  |  2 +-
 config/routes.rb                                   |  2 --
 .../20210502233513_drop_account_tag_stats.rb       | 13 ++++++++
 db/schema.rb                                       | 10 ------
 spec/models/account_tag_stat_spec.rb               | 38 ----------------------
 spec/models/trending_tags_spec.rb                  |  6 ++--
 20 files changed, 59 insertions(+), 160 deletions(-)
 delete mode 100644 app/models/account_tag_stat.rb
 create mode 100644 db/post_migrate/20210502233513_drop_account_tag_stats.rb
 delete mode 100644 spec/models/account_tag_stat_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index f198ad5ba..f28c5b2af 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -6,7 +6,6 @@ class DirectoriesController < ApplicationController
   before_action :authenticate_user!, if: :whitelist_mode?
   before_action :require_enabled!
   before_action :set_instance_presenter
-  before_action :set_tag, only: :show
   before_action :set_accounts
 
   skip_before_action :require_functional!, unless: :whitelist_mode?
@@ -15,23 +14,14 @@ class DirectoriesController < ApplicationController
     render :index
   end
 
-  def show
-    render :index
-  end
-
   private
 
   def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
-  def set_tag
-    @tag = Tag.discoverable.find_normalized!(params[:id])
-  end
-
   def set_accounts
     @accounts = Account.local.discoverable.by_recent_status.page(params[:page]).per(20).tap do |query|
-      query.merge!(Account.tagged_with(@tag.id)) if @tag
       query.merge!(Account.not_excluded_by_account(current_account)) if current_account
     end
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index a1081522e..9f778ffb9 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -22,6 +22,10 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
         visibility: visibility_from_audience
       )
 
+      original_status.tags.each do |tag|
+        tag.use!(@account)
+      end
+
       distribute(@status)
     end
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index c7a655d9d..e46361c14 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -164,7 +164,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
-      TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
+      tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
     end
 
     @mentions.each do |mention|
diff --git a/app/models/account.rb b/app/models/account.rb
index a573365de..994459338 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -111,7 +111,6 @@ class Account < ApplicationRecord
   scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) }
   scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
   scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
-  scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
@@ -279,19 +278,13 @@ class Account < ApplicationRecord
       if hashtags_map.key?(tag.name)
         hashtags_map.delete(tag.name)
       else
-        transaction do
-          tags.delete(tag)
-          tag.decrement_count!(:accounts_count)
-        end
+        tags.delete(tag)
       end
     end
 
     # Add hashtags that were so far missing
     hashtags_map.each_value do |tag|
-      transaction do
-        tags << tag
-        tag.increment_count!(:accounts_count)
-      end
+      tags << tag
     end
   end
 
diff --git a/app/models/account_tag_stat.rb b/app/models/account_tag_stat.rb
deleted file mode 100644
index 3c36c155a..000000000
--- a/app/models/account_tag_stat.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: account_tag_stats
-#
-#  id             :bigint(8)        not null, primary key
-#  tag_id         :bigint(8)        not null
-#  accounts_count :bigint(8)        default(0), not null
-#  hidden         :boolean          default(FALSE), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#
-
-class AccountTagStat < ApplicationRecord
-  belongs_to :tag, inverse_of: :account_tag_stat
-
-  def increment_count!(key)
-    update(key => public_send(key) + 1)
-  end
-
-  def decrement_count!(key)
-    update(key => [public_send(key) - 1, 0].max)
-  end
-end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index efffc7eee..735c30608 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,10 +20,8 @@
 class Tag < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_and_belongs_to_many :accounts
-  has_and_belongs_to_many :sample_accounts, -> { local.discoverable.popular.limit(3) }, class_name: 'Account'
 
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
-  has_one :account_tag_stat, dependent: :destroy
 
   HASHTAG_SEPARATORS = "_\u00B7\u200c"
   HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
@@ -38,29 +36,11 @@ class Tag < ApplicationRecord
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
-  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
-  # Search with case-sensitive to use B-tree index.
-  scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) }
-
-  delegate :accounts_count,
-           :accounts_count=,
-           :increment_count!,
-           :decrement_count!,
-           to: :account_tag_stat
-
-  after_save :save_account_tag_stat
+  scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
   update_index('tags#tag', :self)
 
-  def account_tag_stat
-    super || build_account_tag_stat
-  end
-
-  def cached_sample_accounts
-    Rails.cache.fetch("#{cache_key}/sample_accounts", expires_in: 12.hours) { sample_accounts }
-  end
-
   def to_param
     name
   end
@@ -95,6 +75,10 @@ class Tag < ApplicationRecord
     requested_review_at.present?
   end
 
+  def use!(account, status: nil, at_time: Time.now.utc)
+    TrendingTags.record_use!(self, account, status: status, at_time: at_time)
+  end
+
   def trending?
     TrendingTags.trending?(self)
   end
@@ -127,9 +111,10 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0, options = {})
-      striped_term = term.strip
-      query = Tag.listable.matches_name(striped_term)
-      query = query.merge(matching_name(striped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed]
+      stripped_term = term.strip
+
+      query = Tag.listable.matches_name(stripped_term)
+      query = query.merge(matching_name(stripped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed]
 
       query.order(Arel.sql('length(name) ASC, name ASC'))
            .limit(limit)
@@ -161,11 +146,6 @@ class Tag < ApplicationRecord
 
   private
 
-  def save_account_tag_stat
-    return unless account_tag_stat&.changed?
-    account_tag_stat.save
-  end
-
   def validate_name_change
     errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
   end
diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb
index a9ff5b703..85bfcbea5 100644
--- a/app/models/tag_filter.rb
+++ b/app/models/tag_filter.rb
@@ -33,8 +33,6 @@ class TagFilter
 
   def scope_for(key, value)
     case key.to_s
-    when 'directory'
-      Tag.discoverable
     when 'reviewed'
       Tag.reviewed.order(reviewed_at: :desc)
     when 'unreviewed'
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 9c2aa0ee8..31890b082 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -13,19 +13,23 @@ class TrendingTags
   class << self
     include Redisable
 
-    def record_use!(tag, account, at_time = Time.now.utc)
-      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
+    def record_use!(tag, account, status: nil, at_time: Time.now.utc)
+      return unless tag.usable? && !account.silenced?
 
+      # Even if a tag is not allowed to trend, we still need to
+      # record the stats since they can be displayed in other places
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_use!(tag.id, at_time)
 
-      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      # Only update when the tag was last used once every 12 hours
+      # and only if a status is given (lets use ignore reblogs)
+      tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
     end
 
     def update!(at_time = Time.now.utc)
       tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
-      tags    = Tag.where(id: tag_ids.uniq)
+      tags    = Tag.trendable.where(id: tag_ids.uniq)
 
       # First pass to calculate scores and update the set
 
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index e8e139b05..c42b79db8 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -8,8 +8,7 @@ class ProcessHashtagsService < BaseService
     Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
-
-      TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
+      tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility?
     end
 
     return unless status.distributable?
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 5032397b3..744bdf567 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -35,6 +35,7 @@ class ReblogService < BaseService
 
     create_notification(reblog)
     bump_potential_friendship(account, reblog)
+    record_use(account, reblog)
 
     reblog
   end
@@ -59,6 +60,16 @@ class ReblogService < BaseService
     PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
   end
 
+  def record_use(account, reblog)
+    return unless reblog.public_visibility?
+
+    original_status = reblog.reblog
+
+    original_status.tags.each do |tag|
+      tag.use!(account)
+    end
+  end
+
   def build_json(reblog)
     Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
   end
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
index 287d28e53..adf4ca7b2 100644
--- a/app/views/admin/tags/_tag.html.haml
+++ b/app/views/admin/tags/_tag.html.haml
@@ -10,8 +10,6 @@
         = tag.name
 
         %small
-          = t('admin.tags.in_directory', count: tag.accounts_count)
-          &bull;
           = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
 
           - if tag.trending?
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index d7719d45d..d78f3c6d1 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -5,12 +5,6 @@
   = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
 
 .filters
-  .filter-subset
-    %strong= t('admin.tags.context')
-    %ul
-      %li= filter_link_to t('generic.all'), directory: nil
-      %li= filter_link_to t('admin.tags.directory'), directory: '1'
-
   .filter-subset
     %strong= t('admin.tags.review')
     %ul
@@ -23,8 +17,9 @@
     %strong= t('generic.order_by')
     %ul
       %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
-      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
       %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
+      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
+
 
 = form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
   .fields-group
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
index c9a147587..c4caffda1 100644
--- a/app/views/admin/tags/show.html.haml
+++ b/app/views/admin/tags/show.html.haml
@@ -10,15 +10,6 @@
     %div
       .dashboard__counters__num= number_with_delimiter @accounts_week
       .dashboard__counters__label= t 'admin.tags.accounts_week'
-  %div
-    - if @tag.accounts_count > 0
-      = link_to explore_hashtag_path(@tag) do
-        .dashboard__counters__num= number_with_delimiter @tag.accounts_count
-        .dashboard__counters__label= t 'admin.tags.directory'
-    - else
-      %div
-        .dashboard__counters__num= number_with_delimiter @tag.accounts_count
-        .dashboard__counters__label= t 'admin.tags.directory'
 
 %hr.spacer/
 
@@ -30,8 +21,8 @@
 
   .fields-group
     = f.input :usable, as: :boolean, wrapper: :with_label
-    = f.input :trendable, as: :boolean, wrapper: :with_label, disabled: !Setting.trends
-    = f.input :listable, as: :boolean, wrapper: :with_label, disabled: !Setting.profile_directory
+    = f.input :trendable, as: :boolean, wrapper: :with_label
+    = f.input :listable, as: :boolean, wrapper: :with_label
 
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/config/locales/en.yml b/config/locales/en.yml
index bfa489817..d8ad5bd84 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -698,12 +698,9 @@ en:
       accounts_today: Unique uses today
       accounts_week: Unique uses this week
       breakdown: Breakdown of today's usage by source
-      context: Context
-      directory: In directory
-      in_directory: "%{count} in directory"
-      last_active: Last active
+      last_active: Recently used
       most_popular: Most popular
-      most_recent: Most recent
+      most_recent: Recently created
       name: Hashtag
       review: Review status
       reviewed: Reviewed
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 8ff880ebc..c4388ffc5 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -208,7 +208,7 @@ en:
       rule:
         text: Rule
       tag:
-        listable: Allow this hashtag to appear in searches and on the profile directory
+        listable: Allow this hashtag to appear in searches and suggestions
         name: Hashtag
         trendable: Allow this hashtag to appear under trends
         usable: Allow posts to use this hashtag
diff --git a/config/routes.rb b/config/routes.rb
index 8ca7fccdd..2373d8a51 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -97,8 +97,6 @@ Rails.application.routes.draw do
   post '/interact/:id', to: 'remote_interaction#create'
 
   get '/explore', to: 'directories#index', as: :explore
-  get '/explore/:id', to: 'directories#show', as: :explore_hashtag
-
   get '/settings', to: redirect('/settings/profile')
 
   namespace :settings do
diff --git a/db/post_migrate/20210502233513_drop_account_tag_stats.rb b/db/post_migrate/20210502233513_drop_account_tag_stats.rb
new file mode 100644
index 000000000..80adadcab
--- /dev/null
+++ b/db/post_migrate/20210502233513_drop_account_tag_stats.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class DropAccountTagStats < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    drop_table :account_tag_stats
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 88e906079..19b1afe00 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -115,15 +115,6 @@ ActiveRecord::Schema.define(version: 2021_05_05_174616) do
     t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
   end
 
-  create_table "account_tag_stats", force: :cascade do |t|
-    t.bigint "tag_id", null: false
-    t.bigint "accounts_count", default: 0, null: false
-    t.boolean "hidden", default: false, null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.index ["tag_id"], name: "index_account_tag_stats_on_tag_id", unique: true
-  end
-
   create_table "account_warning_presets", force: :cascade do |t|
     t.text "text", default: "", null: false
     t.datetime "created_at", null: false
@@ -985,7 +976,6 @@ ActiveRecord::Schema.define(version: 2021_05_05_174616) do
   add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_pins", "accounts", on_delete: :cascade
   add_foreign_key "account_stats", "accounts", on_delete: :cascade
-  add_foreign_key "account_tag_stats", "tags", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", on_delete: :nullify
   add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify
diff --git a/spec/models/account_tag_stat_spec.rb b/spec/models/account_tag_stat_spec.rb
deleted file mode 100644
index 6d3057f35..000000000
--- a/spec/models/account_tag_stat_spec.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe AccountTagStat, type: :model do
-  key = 'accounts_count'
-  let(:account_tag_stat) { Fabricate(:tag).account_tag_stat }
-
-  describe '#increment_count!' do
-    it 'calls #update' do
-      args = { key => account_tag_stat.public_send(key) + 1 }
-      expect(account_tag_stat).to receive(:update).with(args)
-      account_tag_stat.increment_count!(key)
-    end
-
-    it 'increments value by 1' do
-      expect do
-        account_tag_stat.increment_count!(key)
-      end.to change { account_tag_stat.accounts_count }.by(1)
-    end
-  end
-
-  describe '#decrement_count!' do
-    it 'calls #update' do
-      args = { key => [account_tag_stat.public_send(key) - 1, 0].max }
-      expect(account_tag_stat).to receive(:update).with(args)
-      account_tag_stat.decrement_count!(key)
-    end
-
-    it 'decrements value by 1' do
-      account_tag_stat.update(key => 1)
-
-      expect do
-        account_tag_stat.decrement_count!(key)
-      end.to change { account_tag_stat.accounts_count }.by(-1)
-    end
-  end
-end
diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb
index b6122c994..dfbc7d6f8 100644
--- a/spec/models/trending_tags_spec.rb
+++ b/spec/models/trending_tags_spec.rb
@@ -7,9 +7,9 @@ RSpec.describe TrendingTags do
 
   describe '.update!' do
     let!(:at_time) { Time.now.utc }
-    let!(:tag1) { Fabricate(:tag, name: 'Catstodon') }
-    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') }
-    let!(:tag3) { Fabricate(:tag, name: 'OCs') }
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
 
     before do
       allow(Redis.current).to receive(:pfcount) do |key|
-- 
cgit 


From 9da5e0b350baed99d8ad81760611dbcdbbb5383d Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Sat, 8 May 2021 23:22:18 +0900
Subject: Fix webfinger_update_due to run WebFinger on stale
 activitypub-account (#16182)

---
 app/services/resolve_account_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index b8ddeb2ad..493995447 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -122,7 +122,7 @@ class ResolveAccountService < BaseService
     return false if @options[:check_delivery_availability] && !DeliveryFailureTracker.available?(@domain)
     return false if @options[:skip_webfinger]
 
-    @account.nil? || (@account.ostatus? && @account.possibly_stale?)
+    @account.nil? || @account.possibly_stale?
   end
 
   def activitypub_ready?
-- 
cgit 


From afb788218913061a36fad9b14e2e3e34025cc25b Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 10 May 2021 17:31:55 +0200
Subject: Fix blocking someone not clearing up list feeds (#16205)

---
 app/lib/feed_manager.rb                   | 30 +++++++++++++++++++++++++++++
 app/services/after_block_service.rb       |  5 +++++
 spec/services/after_block_service_spec.rb | 32 ++++++++++++++++++++++++++-----
 3 files changed, 62 insertions(+), 5 deletions(-)

(limited to 'app/services')

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 43aeecb35..d5e435216 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -194,6 +194,36 @@ class FeedManager
     end
   end
 
+  # Clear all statuses from or mentioning target_account from a list feed
+  # @param [List] list
+  # @param [Account] target_account
+  # @return [void]
+  def clear_from_list(list, target_account)
+    timeline_key        = key(:list, list.id)
+    timeline_status_ids = redis.zrange(timeline_key, 0, -1)
+    statuses            = Status.where(id: timeline_status_ids).select(:id, :reblog_of_id, :account_id).to_a
+    reblogged_ids       = Status.where(id: statuses.map(&:reblog_of_id).compact, account: target_account).pluck(:id)
+    with_mentions_ids   = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact, account: target_account).pluck(:status_id)
+
+    target_statuses = statuses.select do |status|
+      status.account_id == target_account.id || reblogged_ids.include?(status.reblog_of_id) || with_mentions_ids.include?(status.id) || with_mentions_ids.include?(status.reblog_of_id)
+    end
+
+    target_statuses.each do |status|
+      unpush_from_list(list, status)
+    end
+  end
+
+  # Clear all statuses from or mentioning target_account from an account's lists
+  # @param [Account] account
+  # @param [Account] target_account
+  # @return [void]
+  def clear_from_lists(account, target_account)
+    List.where(account: account).each do |list|
+      clear_from_list(list, target_account)
+    end
+  end
+
   # Populate home feed of account from scratch
   # @param [Account] account
   # @return [void]
diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb
index 314919df8..899e84be4 100644
--- a/app/services/after_block_service.rb
+++ b/app/services/after_block_service.rb
@@ -6,6 +6,7 @@ class AfterBlockService < BaseService
     @target_account = target_account
 
     clear_home_feed!
+    clear_list_feeds!
     clear_notifications!
     clear_conversations!
   end
@@ -16,6 +17,10 @@ class AfterBlockService < BaseService
     FeedManager.instance.clear_from_home(@account, @target_account)
   end
 
+  def clear_list_feeds!
+    FeedManager.instance.clear_from_lists(@account, @target_account)
+  end
+
   def clear_conversations!
     AccountConversation.where(account: @account).where('? = ANY(participant_account_ids)', @target_account.id).in_batches.destroy_all
   end
diff --git a/spec/services/after_block_service_spec.rb b/spec/services/after_block_service_spec.rb
index f63b2045a..fe5b26b2b 100644
--- a/spec/services/after_block_service_spec.rb
+++ b/spec/services/after_block_service_spec.rb
@@ -5,12 +5,14 @@ RSpec.describe AfterBlockService, type: :service do
     -> { described_class.new.call(account, target_account) }
   end
 
-  let(:account) { Fabricate(:account) }
-  let(:target_account) { Fabricate(:account) }
+  let(:account)              { Fabricate(:account) }
+  let(:target_account)       { Fabricate(:account) }
+  let(:status)               { Fabricate(:status, account: target_account) }
+  let(:other_status)         { Fabricate(:status, account: target_account) }
+  let(:other_account_status) { Fabricate(:status) }
+  let(:other_account_reblog) { Fabricate(:status, reblog_of_id: other_status.id) }
 
   describe 'home timeline' do
-    let(:status) { Fabricate(:status, account: target_account) }
-    let(:other_account_status) { Fabricate(:status) }
     let(:home_timeline_key) { FeedManager.instance.key(:home, account.id) }
 
     before do
@@ -20,10 +22,30 @@ RSpec.describe AfterBlockService, type: :service do
     it "clears account's statuses" do
       FeedManager.instance.push_to_home(account, status)
       FeedManager.instance.push_to_home(account, other_account_status)
+      FeedManager.instance.push_to_home(account, other_account_reblog)
 
       is_expected.to change {
         Redis.current.zrange(home_timeline_key, 0, -1)
-      }.from([status.id.to_s, other_account_status.id.to_s]).to([other_account_status.id.to_s])
+      }.from([status.id.to_s, other_account_status.id.to_s, other_account_reblog.id.to_s]).to([other_account_status.id.to_s])
+    end
+  end
+
+  describe 'lists' do
+    let(:list)              { Fabricate(:list, account: account) }
+    let(:list_timeline_key) { FeedManager.instance.key(:list, list.id) }
+
+    before do
+      Redis.current.del(list_timeline_key)
+    end
+
+    it "clears account's statuses" do
+      FeedManager.instance.push_to_list(list, status)
+      FeedManager.instance.push_to_list(list, other_account_status)
+      FeedManager.instance.push_to_list(list, other_account_reblog)
+
+      is_expected.to change {
+        Redis.current.zrange(list_timeline_key, 0, -1)
+      }.from([status.id.to_s, other_account_status.id.to_s, other_account_reblog.id.to_s]).to([other_account_status.id.to_s])
     end
   end
 end
-- 
cgit