Switch to static URIs, new URI format in both protocols for new statuses (#4815)

* Decouple Status#local? from uri being nil

* Replace on-the-fly URI generation with stored URIs

- Generate URI in after_save hook for local statuses
- Use static value in TagManager when available, fallback to tag format
- Make TagManager use ActivityPub::TagManager to understand new format
- Adjust tests

* Use other heuristic for locality of old statuses, do not perform long query

* Exclude tombstone stream entries from Atom feed

* Prevent nil statuses from landing in Pubsubhubbub::DistributionWorker

* Fix URI not being saved (#4818)

* Add more specs for Status

* Save generated uri immediately

and also fix method order to minimize diff.

* Fix alternate HTML URL in Atom

* Fix tests

* Remove not-null constraint from statuses migration to speed it up

8 files changed, 51 insertions, 36 deletions

diff --git a/spec/fabricators/status_fabricator.rb b/spec/fabricators/status_fabricator.rb
index 8ec5f4ba7..04bbbcf4b 100644
--- a/spec/fabricators/status_fabricator.rb
+++ b/spec/fabricators/status_fabricator.rb
@@ -1,4 +1,8 @@
 Fabricator(:status) do
   account
   text "Lorem ipsum dolor sit amet"
+
+  after_build do |status|
+    status.uri = Faker::Internet.device_token if !status.account.local? && status.uri.nil?
+  end
 end
diff --git a/spec/lib/activitypub/activity/delete_spec.rb b/spec/lib/activitypub/activity/delete_spec.rb
index 65e743abb..38254e31c 100644
--- a/spec/lib/activitypub/activity/delete_spec.rb
+++ b/spec/lib/activitypub/activity/delete_spec.rb
@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 RSpec.describe ActivityPub::Activity::Delete do
-  let(:sender)    { Fabricate(:account) }
+  let(:sender)    { Fabricate(:account, domain: 'example.com') }
   let(:status)    { Fabricate(:status, account: sender, uri: 'foobar') }
 
   let(:json) do
diff --git a/spec/lib/activitypub/activity/undo_spec.rb b/spec/lib/activitypub/activity/undo_spec.rb
index 4629a033f..14c68efe5 100644
--- a/spec/lib/activitypub/activity/undo_spec.rb
+++ b/spec/lib/activitypub/activity/undo_spec.rb
@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 RSpec.describe ActivityPub::Activity::Undo do
-  let(:sender) { Fabricate(:account) }
+  let(:sender) { Fabricate(:account, domain: 'example.com') }
 
   let(:json) do
     {
diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb
index dfe1d8b8f..ab04ccbab 100644
--- a/spec/lib/formatter_spec.rb
+++ b/spec/lib/formatter_spec.rb
@@ -178,7 +178,7 @@ RSpec.describe Formatter do
     end
 
     context 'with remote status' do
-      let(:status) { Fabricate(:status, text: 'Beep boop', uri: 'beepboop') }
+      let(:status) { Fabricate(:status, account: remote_account, text: 'Beep boop') }
 
       it 'reformats' do
         is_expected.to eq 'Beep boop'
@@ -226,7 +226,7 @@ RSpec.describe Formatter do
     end
 
     context 'with remote status' do
-      let(:status)  { Fabricate(:status, text: '<script>alert("Hello")</script>', uri: 'beep boop') }
+      let(:status)  { Fabricate(:status, account: remote_account, text: '<script>alert("Hello")</script>') }
 
       it 'returns tag-stripped text' do
         is_expected.to eq ''
diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb
index 301a0ce30..0451eceeb 100644
--- a/spec/lib/ostatus/atom_serializer_spec.rb
+++ b/spec/lib/ostatus/atom_serializer_spec.rb
@@ -403,8 +403,7 @@ RSpec.describe OStatus::AtomSerializer do
 
       it 'returns element whose rendered view triggers creation when processed' do
         remote_account = Account.create!(username: 'username')
-        remote_status = Fabricate(:status, account: remote_account)
-        remote_status.stream_entry.update!(created_at: '2000-01-01T00:00:00Z')
+        remote_status = Fabricate(:status, account: remote_account, created_at: '2000-01-01T00:00:00Z')
 
         entry = OStatus::AtomSerializer.new.entry(remote_status.stream_entry, true)
         entry.nodes.delete_if { |node| node[:type] == 'application/activity+json' } # Remove ActivityPub link to simplify test
@@ -421,7 +420,7 @@ RSpec.describe OStatus::AtomSerializer do
 
         ProcessFeedService.new.call(xml, account)
 
-        expect(Status.find_by(uri: "tag:remote,2000-01-01:objectId=#{remote_status.id}:objectType=Status")).to be_instance_of Status
+        expect(Status.find_by(uri: "https://remote/users/#{remote_status.account.to_param}/statuses/#{remote_status.id}")).to be_instance_of Status
       end
     end
 
@@ -465,12 +464,11 @@ RSpec.describe OStatus::AtomSerializer do
     end
 
     it 'appends id element with unique tag' do
-      status = Fabricate(:status, reblog_of_id: nil)
-      status.stream_entry.update!(created_at: '2000-01-01T00:00:00Z')
+      status = Fabricate(:status, reblog_of_id: nil, created_at: '2000-01-01T00:00:00Z')
 
       entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
 
-      expect(entry.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.id}:objectType=Status"
+      expect(entry.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
     end
 
     it 'appends published element with created date' do
@@ -515,7 +513,7 @@ RSpec.describe OStatus::AtomSerializer do
       entry = OStatus::AtomSerializer.new.entry(reblog.stream_entry)
 
       object = entry.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{reblogged.id}:objectType=Status"
+      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{reblogged.account.to_param}/statuses/#{reblogged.id}"
     end
 
     it 'does not append activity:object element if target is not present' do
@@ -532,7 +530,7 @@ RSpec.describe OStatus::AtomSerializer do
 
       link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
       expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/users/username/updates/#{status.stream_entry.id}"
+      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
     end
 
     it 'appends link element for itself' do
@@ -553,7 +551,7 @@ RSpec.describe OStatus::AtomSerializer do
       entry = OStatus::AtomSerializer.new.entry(reply_status.stream_entry)
 
       in_reply_to = entry.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to[:ref]).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{in_reply_to_status.id}:objectType=Status"
+      expect(in_reply_to[:ref]).to eq "https://cb6e6126.ngrok.io/users/#{in_reply_to_status.account.to_param}/statuses/#{in_reply_to_status.id}"
     end
 
     it 'does not append thr:in-reply-to element if not threaded' do
@@ -934,7 +932,7 @@ RSpec.describe OStatus::AtomSerializer do
       favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
 
       object = favourite_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.id}:objectType=Status"
+      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
     end
 
     it 'appends thr:in-reply-to element for status' do
@@ -945,7 +943,7 @@ RSpec.describe OStatus::AtomSerializer do
       favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
 
       in_reply_to = favourite_salmon.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.id}:objectType=Status"
+      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
       expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
     end
 
@@ -1034,7 +1032,7 @@ RSpec.describe OStatus::AtomSerializer do
       unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
 
       object = unfavourite_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.id}:objectType=Status"
+      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
     end
 
     it 'appends thr:in-reply-to element for status' do
@@ -1045,7 +1043,7 @@ RSpec.describe OStatus::AtomSerializer do
       unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
 
       in_reply_to = unfavourite_salmon.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.id}:objectType=Status"
+      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
       expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
     end
 
@@ -1453,7 +1451,7 @@ RSpec.describe OStatus::AtomSerializer do
     it 'appends id element with URL for status' do
       status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
       object = OStatus::AtomSerializer.new.object(status)
-      expect(object.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.id}:objectType=Status"
+      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
     end
 
     it 'appends published element with created date' do
@@ -1463,7 +1461,8 @@ RSpec.describe OStatus::AtomSerializer do
     end
 
     it 'appends updated element with updated date' do
-      status = Fabricate(:status, updated_at: '2000-01-01T00:00:00Z')
+      status = Fabricate(:status)
+      status.updated_at = '2000-01-01T00:00:00Z'
       object = OStatus::AtomSerializer.new.object(status)
       expect(object.updated.text).to eq '2000-01-01T00:00:00Z'
     end
@@ -1523,7 +1522,7 @@ RSpec.describe OStatus::AtomSerializer do
       entry = OStatus::AtomSerializer.new.object(reply)
 
       in_reply_to = entry.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{thread.id}:objectType=Status"
+      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{thread.account.to_param}/statuses/#{thread.id}"
       expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{thread.id}"
     end
 
diff --git a/spec/lib/tag_manager_spec.rb b/spec/lib/tag_manager_spec.rb
index 1fae6bec4..1cd6e0a6f 100644
--- a/spec/lib/tag_manager_spec.rb
+++ b/spec/lib/tag_manager_spec.rb
@@ -157,23 +157,12 @@ RSpec.describe TagManager do
   describe '#uri_for' do
     subject { TagManager.instance.uri_for(target) }
 
-    context 'activity object' do
-      let(:target) { Fabricate(:status, reblog: Fabricate(:status)).stream_entry }
-
-      before { target.update!(created_at: '2000-01-01T00:00:00Z') }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :activity
-        is_expected.to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{target.id}:objectType=Status"
-      end
-    end
-
     context 'comment object' do
       let(:target) { Fabricate(:status, created_at: '2000-01-01T00:00:00Z', reply: true) }
 
       it 'returns the unique tag for status' do
         expect(target.object_type).to eq :comment
-        is_expected.to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{target.id}:objectType=Status"
+        is_expected.to eq target.uri
       end
     end
 
@@ -182,7 +171,7 @@ RSpec.describe TagManager do
 
       it 'returns the unique tag for status' do
         expect(target.object_type).to eq :note
-        is_expected.to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{target.id}:objectType=Status"
+        is_expected.to eq target.uri
       end
     end
 
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index 626fc3f98..484effd5e 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -13,9 +13,15 @@ RSpec.describe Status, type: :model do
     end
 
     it 'returns false if a remote URI is set' do
-      subject.uri = 'a'
+      alice.update(domain: 'example.com')
+      subject.save
       expect(subject.local?).to be false
     end
+
+    it 'returns true if a URI is set and `local` is true' do
+      subject.update(uri: 'example.com', local: true)
+      expect(subject.local?).to be true
+    end
   end
 
   describe '#reblog?' do
@@ -495,7 +501,7 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe 'before_create' do
+  describe 'before_validation' do
     it 'sets account being replied to correctly over intermediary nodes' do
       first_status = Fabricate(:status, account: bob)
       intermediary = Fabricate(:status, thread: first_status, account: alice)
@@ -512,5 +518,22 @@ RSpec.describe Status, type: :model do
       parent = Fabricate(:status, text: 'First')
       expect(Status.create(account: alice, thread: parent, text: 'Response').conversation_id).to eq parent.conversation_id
     end
+
+    it 'sets `local` to true for status by local account' do
+      expect(Status.create(account: alice, text: 'foo').local).to be true
+    end
+
+    it 'sets `local` to false for status by remote account' do
+      alice.update(domain: 'example.com')
+      expect(Status.create(account: alice, text: 'foo').local).to be false
+    end
+  end
+
+  describe 'after_create' do
+    it 'saves ActivityPub uri as uri for local status' do
+      status = Status.create(account: alice, text: 'foo')
+      status.reload
+      expect(status.uri).to start_with('https://')
+    end
   end
 end
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 3a0786d03..b0aa740ac 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -55,7 +55,7 @@ RSpec.describe FetchLinkCardService do
   end
 
   context 'in a remote status' do
-    let(:status) { Fabricate(:status, uri: 'abc', text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
+    let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
 
     it 'parses out URLs' do
       expect(a_request(:head, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once