[Federation, Revise] Add simple retries to ActivityPub::ProcessCollectionItemsService

6 files changed, 17 insertions, 4 deletions

diff --git a/app/lib/activitypub/activity/add.rb b/app/lib/activitypub/activity/add.rb
index 688ab00b3..03b584302 100644
--- a/app/lib/activitypub/activity/add.rb
+++ b/app/lib/activitypub/activity/add.rb
@@ -9,6 +9,6 @@ class ActivityPub::Activity::Add < ActivityPub::Activity
 
     return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
 
-    StatusPin.create!(account: @account, status: status)
+    StatusPin.create(account: @account, status: status)
   end
 end
diff --git a/app/models/collection_item.rb b/app/models/collection_item.rb
index cecb86534..24aaf66d4 100644
--- a/app/models/collection_item.rb
+++ b/app/models/collection_item.rb
@@ -7,6 +7,7 @@
 #  account_id :bigint(8)
 #  uri        :string           not null
 #  processed  :boolean          default(FALSE), not null
+#  retries    :integer          default(0), not null
 #
 
 class CollectionItem < ApplicationRecord
diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index a77c03bed..0a20f5edc 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -44,7 +44,7 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
     StatusPin.where(account: @account, status_id: to_remove).delete_all unless to_remove.empty?
 
     to_add.each do |status_id|
-      StatusPin.create!(account: @account, status_id: status_id)
+      StatusPin.create(account: @account, status_id: status_id)
     end
   end
 
diff --git a/app/services/activitypub/process_collection_items_service.rb b/app/services/activitypub/process_collection_items_service.rb
index 936593166..9c30d81e9 100644
--- a/app/services/activitypub/process_collection_items_service.rb
+++ b/app/services/activitypub/process_collection_items_service.rb
@@ -5,12 +5,18 @@ class ActivityPub::ProcessCollectionItemsService < BaseService
     RedisLock.acquire(lock_options(account_id)) do |lock|
       if lock.acquired?
         CollectionItem.unprocessed.where(account_id: account_id).find_each do |item|
+          # Avoid failing servers holding up the rest of the queue.
+          next if item.retries.positive? && rand(3).positive?
+
           begin
             FetchRemoteStatusService.new.call(item.uri, nil, on_behalf_of)
           rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotFound
             nil
+          rescue HTTP::TimeoutError
+            item.increment!(:retries)
           end
-          item.update!(processed: true)
+
+          item.update!(processed: true) if item.retries.zero? || item.retries > 4
         end
       end
     end
diff --git a/db/migrate/20200821051721_add_retries_to_collection_items.rb b/db/migrate/20200821051721_add_retries_to_collection_items.rb
new file mode 100644
index 000000000..9cee437d9
--- /dev/null
+++ b/db/migrate/20200821051721_add_retries_to_collection_items.rb
@@ -0,0 +1,5 @@
+class AddRetriesToCollectionItems < ActiveRecord::Migration[5.2]
+  def change
+    add_column :collection_items, :retries, :integer, limit: 1, default: 0, null: false
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 267a55316..22210f7a2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_08_18_160106) do
+ActiveRecord::Schema.define(version: 2020_08_21_051721) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -298,6 +298,7 @@ ActiveRecord::Schema.define(version: 2020_08_18_160106) do
     t.bigint "account_id"
     t.string "uri", null: false
     t.boolean "processed", default: false, null: false
+    t.integer "retries", limit: 2, default: 0, null: false
     t.index ["account_id"], name: "index_collection_items_on_account_id"
     t.index ["account_id"], name: "unprocessed_collection_item_account_ids", where: "(processed = false)"
     t.index ["id"], name: "unprocessed_collection_item_ids", order: :desc, where: "(processed = false)"