Add soft delete for statuses for instant deletes through API (#11623)

* Add soft delete for statuses to allow them to appear instant

* Allow reporting soft-deleted statuses and show them in the admin UI

* Change index for getting an account's statuses

14 files changed, 42 insertions, 8 deletions

diff --git a/Gemfile b/Gemfile
index 250a28a3a..86dab965a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -43,6 +43,7 @@ gem 'omniauth-cas', '~> 1.1'
 gem 'omniauth-saml', '~> 1.10'
 gem 'omniauth', '~> 1.9'
 
+gem 'discard', '~> 1.1'
 gem 'doorkeeper', '~> 5.1'
 gem 'fast_blank', '~> 1.0'
 gem 'fastimage'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1da6d73a6..b896909a4 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -204,6 +204,8 @@ GEM
       devise (>= 4.0.0)
       rpam2 (~> 4.0)
     diff-lcs (1.3)
+    discard (1.1.0)
+      activerecord (>= 4.2, < 7)
     docile (1.3.2)
     domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
@@ -692,6 +694,7 @@ DEPENDENCIES
   devise (~> 4.6)
   devise-two-factor (~> 3.1)
   devise_pam_authenticatable2 (~> 9.2)
+  discard (~> 1.1)
   doorkeeper (~> 5.1)
   dotenv-rails (~> 2.7)
   fabrication (~> 2.20)
diff --git a/app/controllers/api/v1/reports_controller.rb b/app/controllers/api/v1/reports_controller.rb
index e182a9c6c..1b0b4b05b 100644
--- a/app/controllers/api/v1/reports_controller.rb
+++ b/app/controllers/api/v1/reports_controller.rb
@@ -21,7 +21,7 @@ class Api::V1::ReportsController < Api::BaseController
   private
 
   def reported_status_ids
-    reported_account.statuses.find(status_ids).pluck(:id)
+    reported_account.statuses.with_discarded.find(status_ids).pluck(:id)
   end
 
   def status_ids
diff --git a/app/controllers/api/v1/statuses/reblogs_controller.rb b/app/controllers/api/v1/statuses/reblogs_controller.rb
index ed4f55100..42381a37f 100644
--- a/app/controllers/api/v1/statuses/reblogs_controller.rb
+++ b/app/controllers/api/v1/statuses/reblogs_controller.rb
@@ -18,6 +18,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
     @reblogs_map = { @status.id => false }
 
     authorize status_for_destroy, :unreblog?
+    status_for_destroy.discard
     RemovalWorker.perform_async(status_for_destroy.id)
 
     render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, reblogs_map: @reblogs_map)
@@ -30,7 +31,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   end
 
   def status_for_destroy
-    current_user.account.statuses.where(reblog_of_id: params[:status_id]).first!
+    @status_for_destroy ||= current_user.account.statuses.where(reblog_of_id: params[:status_id]).first!
   end
 
   def reblog_params
diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 39ca56482..bba3c0651 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -53,6 +53,7 @@ class Api::V1::StatusesController < Api::BaseController
     @status = Status.where(account_id: current_user.account).find(params[:id])
     authorize @status, :destroy?
 
+    @status.discard
     RemovalWorker.perform_async(@status.id, redraft: true)
 
     render json: @status, serializer: REST::StatusSerializer, source_requested: true
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index 831d8b7c5..e09cc2594 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -34,6 +34,7 @@ class Form::StatusBatch
 
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
+      status.discard
       RemovalWorker.perform_async(status.id, redraft: false)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
diff --git a/app/models/report.rb b/app/models/report.rb
index 5192ceef7..1e707ff1c 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -43,7 +43,7 @@ class Report < ApplicationRecord
   end
 
   def statuses
-    Status.where(id: status_ids).includes(:account, :media_attachments, :mentions)
+    Status.with_discarded.where(id: status_ids).includes(:account, :media_attachments, :mentions)
   end
 
   def media_attachments
diff --git a/app/models/status.rb b/app/models/status.rb
index 0538c4e9e..9cfaddcec 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -22,15 +22,19 @@
 #  application_id         :bigint(8)
 #  in_reply_to_account_id :bigint(8)
 #  poll_id                :bigint(8)
+#  deleted_at             :datetime
 #
 
 class Status < ApplicationRecord
   before_destroy :unlink_from_conversations
 
+  include Discard::Model
   include Paginable
   include Cacheable
   include StatusThreadingConcern
 
+  self.discard_column = :deleted_at
+
   # If `override_timestamps` is set at creation time, Snowflake ID creation
   # will be based on current time instead of `created_at`
   attr_accessor :override_timestamps
@@ -72,7 +76,7 @@ class Status < ApplicationRecord
 
   accepts_nested_attributes_for :poll
 
-  default_scope { recent }
+  default_scope { recent.kept }
 
   scope :recent, -> { reorder(id: :desc) }
   scope :remote, -> { where(local: false).where.not(uri: nil) }
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index 9376db7ff..6facc0a56 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -16,11 +16,14 @@
         - video = status.proper.media_attachments.first
         = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description
       - else
-        = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
+        = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
       = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+      - if status.discarded?
+        ·
+        %span.negative-hint= t('admin.statuses.deleted')
       ·
       - if status.reblog?
         = fa_icon('retweet fw')
diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb
index 14423a4fb..2a1eaa89b 100644
--- a/app/workers/removal_worker.rb
+++ b/app/workers/removal_worker.rb
@@ -4,7 +4,7 @@ class RemovalWorker
   include Sidekiq::Worker
 
   def perform(status_id, options = {})
-    RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys)
+    RemoveStatusService.new.call(Status.with_discarded.find(status_id), **options.symbolize_keys)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 8d267065c..a50dcb8a5 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -499,6 +499,7 @@ en:
         delete: Delete
         nsfw_off: Mark as not sensitive
         nsfw_on: Mark as sensitive
+      deleted: Deleted
       failed_to_execute: Failed to execute
       media:
         title: Media
diff --git a/db/migrate/20190819134503_add_deleted_at_to_statuses.rb b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb
new file mode 100644
index 000000000..5af109097
--- /dev/null
+++ b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb
@@ -0,0 +1,5 @@
+class AddDeletedAtToStatuses < ActiveRecord::Migration[5.2]
+  def change
+    add_column :statuses, :deleted_at, :datetime
+  end
+end
diff --git a/db/migrate/20190820003045_update_statuses_index.rb b/db/migrate/20190820003045_update_statuses_index.rb
new file mode 100644
index 000000000..5c2ea1f6a
--- /dev/null
+++ b/db/migrate/20190820003045_update_statuses_index.rb
@@ -0,0 +1,13 @@
+class UpdateStatusesIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], where: 'deleted_at IS NULL', order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20190820 }
+    remove_index :statuses, name: :index_statuses_20180106
+  end
+
+  def down
+    safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20180106 }
+    remove_index :statuses, name: :index_statuses_20190820
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 18f615d61..afa6d724c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_15_225426) do
+ActiveRecord::Schema.define(version: 2019_08_20_003045) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -644,7 +644,8 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do
     t.bigint "application_id"
     t.bigint "in_reply_to_account_id"
     t.bigint "poll_id"
-    t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20180106", order: { id: :desc }
+    t.datetime "deleted_at"
+    t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"