Monsterfork v1-style curated Federated timeline

10 files changed, 95 insertions, 30 deletions

diff --git a/app/controllers/api/v1/statuses/bookmarks_controller.rb b/app/controllers/api/v1/statuses/bookmarks_controller.rb
index 19963c002..1bf698a78 100644
--- a/app/controllers/api/v1/statuses/bookmarks_controller.rb
+++ b/app/controllers/api/v1/statuses/bookmarks_controller.rb
@@ -9,6 +9,7 @@ class Api::V1::Statuses::BookmarksController < Api::BaseController
 
   def create
     current_account.bookmarks.find_or_create_by!(account: current_account, status: @status)
+    curate! unless @status.curated? || !@status.published?
     render json: @status, serializer: REST::StatusSerializer
   end
 
@@ -37,4 +38,9 @@ class Api::V1::Statuses::BookmarksController < Api::BaseController
   rescue Mastodon::NotPermittedError
     not_found
   end
+
+  def curate!
+    @status.curate!
+    DistributionWorker.perform_async(@status.id)
+  end
 end
diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb
index 2839da5cb..afde8aa65 100644
--- a/app/models/public_feed.rb
+++ b/app/models/public_feed.rb
@@ -26,7 +26,8 @@ class PublicFeed < Feed
     scope.merge!(without_replies_scope) unless with_replies?
     scope.merge!(without_reblogs_scope) unless with_reblogs?
     scope.merge!(local_only_scope) if local_only?
-    scope.merge!(remote_only_scope) if remote_only?
+    #scope.merge!(remote_only_scope) if remote_only?
+    scope.merge!(curated_scope) unless remote_only?
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
 
@@ -79,6 +80,10 @@ class PublicFeed < Feed
     Status.remote
   end
 
+  def curated_scope
+    Status.curated
+  end
+
   def without_replies_scope
     Status.without_replies
   end
diff --git a/app/models/status.rb b/app/models/status.rb
index c79cbeaf9..ba8b4ff2b 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -35,6 +35,7 @@
 #  expires_at             :datetime
 #  publish_at             :datetime
 #  originally_local_only  :boolean          default(FALSE), not null
+#  curated                :boolean          default(FALSE), not null
 #
 
 # rubocop:disable Metrics/ClassLength
@@ -142,6 +143,7 @@ class Status < ApplicationRecord
   scope :replies, -> { where(reply: true) }
   scope :expired, -> { published.where('statuses.expires_at IS NOT NULL AND statuses.expires_at < ?', Time.now.utc) }
   scope :ready_to_publish, -> { unpublished.where('statuses.publish_at IS NOT NULL AND statuses.publish_at < ?', Time.now.utc) }
+  scope :curated, -> { where(curated: true) }
 
   scope :not_hidden_by_account, ->(account) do
     left_outer_joins(:mutes, :conversation_mute).where('(status_mutes.account_id IS NULL OR status_mutes.account_id != ?) AND (conversation_mutes.account_id IS NULL OR conversation_mutes.account_id != ?)', account.id, account.id)
@@ -308,6 +310,14 @@ class Status < ApplicationRecord
     update_status_stat!(key => [public_send(key) - 1, 0].max)
   end
 
+  def curate!
+    update_column(:curated, true) if public_visibility? && !curated
+  end
+
+  def uncurate!
+    update_column(:curated, false) if curated
+  end
+
   def notify=(value)
     Redis.current.set("status:#{id}:notify", value ? 1 : 0, ex: 1.hour)
     @notify = value
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 5b5d2c0a1..1fa8b2520 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -22,23 +22,18 @@ class FanOutOnWriteService < BaseService
     end
 
     return if status.account.silenced?
+    return if status.reblog? && !Setting.show_reblogs_in_public_timelines
 
-    render_anonymous_payload(status.proper)
-    deliver_to_hashtags(status)
-
-    if status.reblog?
-      if status.local? && status.reblog.public_visibility? && !status.reblog.account.silenced?
-        deliver_to_public(status.reblog)
-        deliver_to_media(status.reblog) if status.reblog.media_attachments.any?
-      end
-      return
+    if status.distributable?
+      render_anonymous_payload(status)
+      deliver_to_hashtags(status)
     end
 
-    deliver_to_hashtags(status) if status.distributable?
-    return if !status.public_visibility? || (status.reply? && status.in_reply_to_account_id != status.account_id)
+    return unless status.public_visibility?
+    return if status.reply? && status.in_reply_to_account_id != status.account_id && !Setting.show_replies_in_public_timelines
 
-    deliver_to_media(status, true) if status.media_attachments.any?
-    deliver_to_public(status, true)
+    deliver_to_public(status)
+    deliver_to_media(status) if status.media_attachments.any?
   end
 
   private
@@ -93,26 +88,18 @@ class FanOutOnWriteService < BaseService
     end
   end
 
-  def deliver_to_public(status, tavern = false)
-    key = "timeline:public:#{status.id}"
-    return if Redis.current.get(key)
-
+  def deliver_to_public(status)
     Rails.logger.debug "Delivering status #{status.id} to public timeline"
 
-    Redis.current.set(key, 1, ex: 2.hours)
-
-    Redis.current.publish('timeline:public', @payload) if status.local? || !tavern
+    Redis.current.publish('timeline:public', @payload) if status.curated?
     Redis.current.publish('timeline:public:local', @payload) if status.local?
     Redis.current.publish('timeline:public:remote', @payload)
   end
 
-  def deliver_to_media(status, tavern = false)
-    key = "timeline:public:#{status.id}"
-    return if Redis.current.get(key)
-
+  def deliver_to_media(status)
     Rails.logger.debug "Delivering status #{status.id} to media timeline"
 
-    Redis.current.publish('timeline:public:media', @payload) if status.local? || !tavern
+    Redis.current.publish('timeline:public:media', @payload) if status.curated?
     Redis.current.publish('timeline:public:local:media', @payload) if status.local?
     Redis.current.publish('timeline:public:remote:media', @payload)
   end
diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb
index 02b26458a..af08f5267 100644
--- a/app/services/favourite_service.rb
+++ b/app/services/favourite_service.rb
@@ -16,6 +16,7 @@ class FavouriteService < BaseService
     return favourite unless favourite.nil?
 
     favourite = Favourite.create!(account: account, status: status)
+    curate!(status) unless status.curated? || !status.published?
 
     create_notification(favourite)
     bump_potential_friendship(account, status)
@@ -44,4 +45,9 @@ class FavouriteService < BaseService
   def build_json(favourite)
     Oj.dump(serialize_payload(favourite, ActivityPub::LikeSerializer))
   end
+
+  def curate!(status)
+    status.curate!
+    DistributionWorker.perform_async(status.id)
+  end
 end
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index ddd22e379..86b37560a 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -29,6 +29,7 @@ class ReblogService < BaseService
     end
 
     reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit], sensitive: true, spoiler_text: options[:spoiler_text] || '', published: true)
+    curate!(reblogged_status) unless reblogged_status.curated? || !reblogged_status.published?
 
     DistributionWorker.perform_async(reblog.id)
     ActivityPub::DistributionWorker.perform_async(reblog.id) unless reblogged_status.local_only?
@@ -62,4 +63,9 @@ class ReblogService < BaseService
   def build_json(reblog)
     Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog, embed: false), ActivityPub::ActivitySerializer, signer: reblog.account, target_domain: reblog.account.domain))
   end
+
+  def curate!(status)
+    status.curate!
+    DistributionWorker.perform_async(status.id)
+  end
 end
diff --git a/db/migrate/20200921024447_add_curated_to_statuses.rb b/db/migrate/20200921024447_add_curated_to_statuses.rb
new file mode 100644
index 000000000..05558ded3
--- /dev/null
+++ b/db/migrate/20200921024447_add_curated_to_statuses.rb
@@ -0,0 +1,7 @@
+class AddCuratedToStatuses < ActiveRecord::Migration[5.2]
+  def change
+    safety_assured do
+      add_column :statuses, :curated, :boolean, default: false, null: false
+    end
+  end
+end
diff --git a/db/migrate/20200921030158_backfill_curated_statuses.rb b/db/migrate/20200921030158_backfill_curated_statuses.rb
new file mode 100644
index 000000000..f9bf32afb
--- /dev/null
+++ b/db/migrate/20200921030158_backfill_curated_statuses.rb
@@ -0,0 +1,12 @@
+class BackfillCuratedStatuses < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    Status.with_public_visibility.joins(:status_stat).where('favourites_count != 0 OR reblogs_count != 0').in_batches.update_all(curated: true)
+    Status.with_public_visibility.where(curated: false).left_outer_joins(:bookmarks).where.not(bookmarks: { status_id: nil }).in_batches.update_all(curated: true)
+  end
+
+  def down
+    nil
+  end
+end
diff --git a/db/migrate/20200923000000_update_status_indexes_202009.rb b/db/migrate/20200923000000_update_status_indexes_202009.rb
new file mode 100644
index 000000000..9b6f58d3c
--- /dev/null
+++ b/db/migrate/20200923000000_update_status_indexes_202009.rb
@@ -0,0 +1,25 @@
+class UpdateStatusIndexes202009 < ActiveRecord::Migration[5.2]
+  def up
+    safety_assured do
+      remove_index :statuses, name: "index_statuses_local"
+      remove_index :statuses, name: "index_statuses_local_reblogs"
+      remove_index :statuses, name: "index_statuses_public"
+
+      add_index :statuses, :id, name: "index_statuses_local", order: { id: :desc }, where: "(published = TRUE) AND (local = TRUE OR (uri IS NULL)) AND (deleted_at IS NULL)"
+      add_index :statuses, :id, name: "index_statuses_curated", order: { id: :desc }, where: "(published = TRUE) AND (deleted_at IS NULL) AND (curated = TRUE)"
+      add_index :statuses, :id, name: "index_statuses_public", order: { id: :desc }, where: "(published = TRUE) AND (deleted_at IS NULL)"
+    end
+  end
+
+  def down
+    safety_assured do
+      remove_index :statuses, name: "index_statuses_local"
+      remove_index :statuses, name: "index_statuses_curated"
+      remove_index :statuses, name: "index_statuses_public"
+
+      add_index :statuses, ["id", "account_id"], name: "index_statuses_local", order: { id: :desc }, where: "((published = TRUE) AND (local = TRUE OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((reply = FALSE) OR (in_reply_to_account_id = account_id)))"
+      add_index :statuses, ["id", "account_id"], name: "index_statuses_local_reblogs", where: "(((local = TRUE) OR (uri IS NULL)) AND (statuses.reblog_of_id IS NOT NULL))"
+      add_index :statuses, ["id", "account_id"], name: "index_statuses_public", order: { id: :desc }, where: "((published = TRUE) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((reply = FALSE) OR (in_reply_to_account_id = account_id)))"
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index eb0605314..7a5411d1c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_09_20_084007) do
+ActiveRecord::Schema.define(version: 2020_09_21_030158) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -883,14 +883,15 @@ ActiveRecord::Schema.define(version: 2020_09_20_084007) do
     t.datetime "expires_at"
     t.datetime "publish_at"
     t.boolean "originally_local_only", default: false, null: false
+    t.boolean "curated", default: false, null: false
     t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
     t.index ["account_id", "id"], name: "index_unpublished_statuses", order: { id: :desc }, where: "((deleted_at IS NULL) AND (published = false))"
     t.index ["conversation_id"], name: "index_statuses_on_conversation_id", where: "(deleted_at IS NULL)"
     t.index ["expires_at"], name: "index_statuses_on_expires_at", where: "(expires_at IS NOT NULL)"
-    t.index ["id", "account_id"], name: "index_statuses_local", order: { id: :desc }, where: "((published = true) AND ((local = true) OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((reply = false) OR (in_reply_to_account_id = account_id)))"
-    t.index ["id", "account_id"], name: "index_statuses_local_reblogs", where: "(((local = true) OR (uri IS NULL)) AND (reblog_of_id IS NOT NULL))"
     t.index ["id", "account_id"], name: "index_statuses_on_id_and_account_id"
-    t.index ["id", "account_id"], name: "index_statuses_public", order: { id: :desc }, where: "((published = true) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((reply = false) OR (in_reply_to_account_id = account_id)))"
+    t.index ["id"], name: "index_statuses_curated", order: :desc, where: "((published = true) AND (deleted_at IS NULL) AND (curated = true))"
+    t.index ["id"], name: "index_statuses_local", order: :desc, where: "((published = true) AND ((local = true) OR (uri IS NULL)) AND (deleted_at IS NULL))"
+    t.index ["id"], name: "index_statuses_public", order: :desc, where: "((published = true) AND (deleted_at IS NULL))"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
     t.index ["publish_at"], name: "index_statuses_on_publish_at", where: "(publish_at IS NOT NULL)"