Fixes/do not override timestamps (#7336)

* Revert "Fixes/do not override timestamps (#7331)"

This reverts commit 581a5c9d29ef2a12f46b67a1097a9ad6df1c6953.

* Document Snowflake ID corner-case a bit more

Snowflake IDs are used for two purposes: making object identifiers harder to
guess and ensuring they are in chronological order. For this reason, they
are based on the `created_at` attribute of the object.

Unfortunately, inserting items with older snowflakes IDs will break the
assumption of consumers of the paging APIs that new items will always have
a greater identifier than the last seen one.

* Add `override_timestamps` virtual attribute to not correlate snowflake ID with created_at

8 files changed, 23 insertions, 7 deletions

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 5b97a6208..84d4b1752 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -78,9 +78,11 @@ class ActivityPub::Activity
     notify_about_reblog(status) if reblog_of_local_account?(status)
     notify_about_mentions(status)
 
-    # Only continue if the status is supposed to have
-    # arrived in real-time
-    return unless status.within_realtime_window?
+    # Only continue if the status is supposed to have arrived in real-time.
+    # Note that if @options[:override_timestamps] isn't set, the status
+    # may have a lower snowflake id than other existing statuses, potentially
+    # "hiding" it from paginated API calls
+    return unless @options[:override_timestamps] || status.within_realtime_window?
 
     distribute_to_followers(status)
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 8840a450c..7e146ea8c 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -16,6 +16,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       reblog: original_status,
       uri: @json['id'],
       created_at: @json['published'],
+      override_timestamps: @options[:override_timestamps],
       visibility: original_status.visibility
     )
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index edee2691f..8d17a4ebe 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -48,6 +48,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       language: detected_language,
       spoiler_text: @object['summary'] || '',
       created_at: @object['published'],
+      override_timestamps: @options[:override_timestamps],
       reply: @object['inReplyTo'].present?,
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index a24a0093c..1e7f47029 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -40,6 +40,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
         text: content,
         spoiler_text: content_warning,
         created_at: published,
+        override_timestamps: @options[:override_timestamps],
         reply: thread?,
         language: content_language,
         visibility: visibility_scope,
@@ -61,7 +62,14 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
 
     LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-    DistributionWorker.perform_async(status.id) if status.within_realtime_window?
+
+    # Only continue if the status is supposed to have arrived in real-time.
+    # Note that if @options[:override_timestamps] isn't set, the status
+    # may have a lower snowflake id than other existing statuses, potentially
+    # "hiding" it from paginated API calls
+    return status unless @options[:override_timestamps] || status.within_realtime_window?
+
+    DistributionWorker.perform_async(status.id)
 
     status
   end
diff --git a/app/models/status.rb b/app/models/status.rb
index fbb1f89aa..853e75b43 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -31,6 +31,10 @@ class Status < ApplicationRecord
   include Cacheable
   include StatusThreadingConcern
 
+  # If `override_timestamps` is set at creation time, Snowflake ID creation
+  # will be based on current time instead of `created_at`
+  attr_accessor :override_timestamps
+
   update_index('statuses#status', :proper) if Chewy.enabled?
 
   enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility
diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb
index bb9adf64b..0e2e0eddd 100644
--- a/app/workers/activitypub/processing_worker.rb
+++ b/app/workers/activitypub/processing_worker.rb
@@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id))
+    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc..978c3aba2 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -6,6 +6,6 @@ class ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id))
+    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end
diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb
index 219e323d4..9e5bc7383 100644
--- a/lib/mastodon/snowflake.rb
+++ b/lib/mastodon/snowflake.rb
@@ -7,7 +7,7 @@ module Mastodon::Snowflake
     def self.around_create(record)
       now = Time.now.utc
 
-      if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at
+      if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at || record.override_timestamps
         yield
       else
         record.id = Mastodon::Snowflake.id_at(record.created_at)