From 4ec1771165ab8dd40e52804fd087eacfab25290b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 28 Sep 2017 15:31:31 +0200
Subject: Add ability to specify alternative text for media attachments (#5123)

* Fix #117 - Add ability to specify alternative text for media attachments

- POST /api/v1/media accepts `description` straight away
- PUT /api/v1/media/:id to update `description` (only for unattached ones)
- Serialized as `name` of Document object in ActivityPub
- Uploads form adjusted for better performance and description input

* Add tests

* Change undo button blend mode to difference
---
 app/lib/activitypub/activity/create.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 4e19b3096..55addd66e 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -105,7 +105,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       next if unsupported_media_type?(attachment['mediaType']) || attachment['url'].blank?
 
       href             = Addressable::URI.parse(attachment['url']).normalize.to_s
-      media_attachment = MediaAttachment.create(status: status, account: status.account, remote_url: href)
+      media_attachment = MediaAttachment.create(status: status, account: status.account, remote_url: href, description: attachment['name'].presence)
 
       next if skip_download?
 
-- 
cgit 


From 63f097979990bf5ba9db848b8a253056bad781af Mon Sep 17 00:00:00 2001
From: Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
Date: Wed, 4 Oct 2017 08:13:48 +0900
Subject: Validate id of ActivityPub representations (#5114)

Additionally, ActivityPub::FetchRemoteStatusService no longer parses
activities.
OStatus::Activity::Creation no longer delegates to ActivityPub because
the provided ActivityPub representations are not signed while OStatus
representations are.
---
 app/controllers/concerns/signature_verification.rb |  2 +-
 app/helpers/jsonld_helper.rb                       | 13 ++++++-
 app/lib/activitypub/activity/announce.rb           |  2 +-
 app/lib/activitypub/activity/create.rb             |  2 +-
 app/lib/activitypub/linked_data_signature.rb       |  2 +-
 app/lib/ostatus/activity/creation.rb               |  9 -----
 .../activitypub/fetch_remote_account_service.rb    | 10 ++++--
 .../activitypub/fetch_remote_key_service.rb        | 25 +++++++++----
 .../activitypub/fetch_remote_status_service.rb     | 37 +++++++++----------
 .../activitypub/process_account_service.rb         |  6 ++--
 app/services/fetch_atom_service.rb                 | 13 +++----
 app/services/fetch_remote_account_service.rb       | 14 ++++----
 app/services/fetch_remote_status_service.rb        | 16 ++++-----
 app/services/resolve_remote_account_service.rb     |  2 +-
 spec/helpers/jsonld_helper_spec.rb                 | 35 +++++++++++++++++-
 .../fetch_remote_account_service_spec.rb           |  2 +-
 .../fetch_remote_status_service_spec.rb            | 41 +---------------------
 17 files changed, 118 insertions(+), 113 deletions(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb
index dc2d9a678..2baafb5bf 100644
--- a/app/controllers/concerns/signature_verification.rb
+++ b/app/controllers/concerns/signature_verification.rb
@@ -117,7 +117,7 @@ module SignatureVerification
       ResolveRemoteAccountService.new.call(key_id.gsub(/\Aacct:/, ''))
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
       account   = ActivityPub::TagManager.instance.uri_to_resource(key_id, Account)
-      account ||= ActivityPub::FetchRemoteKeyService.new.call(key_id)
+      account ||= ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false)
       account
     end
   end
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index d82a07332..c23a2e095 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -22,7 +22,18 @@ module JsonLdHelper
     graph.dump(:normalize)
   end
 
-  def fetch_resource(uri)
+  def fetch_resource(uri, id)
+    unless id
+      json = fetch_resource_without_id_validation(uri)
+      return unless json
+      uri = json['id']
+    end
+
+    json = fetch_resource_without_id_validation(uri)
+    json.present? && json['id'] == uri ? json : nil
+  end
+
+  def fetch_resource_without_id_validation(uri)
     response = build_request(uri).perform
     return if response.code != 200
     body_to_json(response.to_s)
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 4516454e1..1cf844281 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -27,7 +27,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
 
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri)
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true)
     elsif @object['url'].present?
       ::FetchRemoteStatusService.new.call(@object['url'])
     end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 55addd66e..be656de48 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -80,7 +80,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if tag['href'].blank?
 
     account = account_from_uri(tag['href'])
-    account = FetchRemoteAccountService.new.call(tag['href']) if account.nil?
+    account = FetchRemoteAccountService.new.call(tag['href'], id: false) if account.nil?
     return if account.nil?
     account.mentions.create(status: status)
   end
diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb
index adb8b6cdf..16142a6ff 100644
--- a/app/lib/activitypub/linked_data_signature.rb
+++ b/app/lib/activitypub/linked_data_signature.rb
@@ -19,7 +19,7 @@ class ActivityPub::LinkedDataSignature
     return unless type == 'RsaSignature2017'
 
     creator   = ActivityPub::TagManager.instance.uri_to_resource(creator_uri, Account)
-    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri)
+    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false)
 
     return if creator.nil?
 
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 2687776f9..511c462d4 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -9,11 +9,6 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
 
     return [nil, false] if @account.suspended?
 
-    if activitypub_uri? && [:public, :unlisted].include?(visibility_scope)
-      result = perform_via_activitypub
-      return result if result.first.present?
-    end
-
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         # Return early if status already exists in db
@@ -66,10 +61,6 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     status
   end
 
-  def perform_via_activitypub
-    [find_status(activitypub_uri) || ActivityPub::FetchRemoteStatusService.new.call(activitypub_uri), false]
-  end
-
   def content
     @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content
   end
diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb
index cb6e40748..e6c6338be 100644
--- a/app/services/activitypub/fetch_remote_account_service.rb
+++ b/app/services/activitypub/fetch_remote_account_service.rb
@@ -5,14 +5,18 @@ class ActivityPub::FetchRemoteAccountService < BaseService
 
   # Should be called when uri has already been checked for locality
   # Does a WebFinger roundtrip on each call
-  def call(uri, prefetched_json = nil)
-    @json = body_to_json(prefetched_json) || fetch_resource(uri)
+  def call(uri, id: true, prefetched_body: nil)
+    @json = if prefetched_body.nil?
+              fetch_resource(uri, id)
+            else
+              body_to_json(prefetched_body)
+            end
 
     return unless supported_context? && expected_type?
 
     @uri      = @json['id']
     @username = @json['preferredUsername']
-    @domain   = Addressable::URI.parse(uri).normalized_host
+    @domain   = Addressable::URI.parse(@uri).normalized_host
 
     return unless verified_webfinger?
 
diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb
index ebd64071e..ce1048fee 100644
--- a/app/services/activitypub/fetch_remote_key_service.rb
+++ b/app/services/activitypub/fetch_remote_key_service.rb
@@ -4,13 +4,26 @@ class ActivityPub::FetchRemoteKeyService < BaseService
   include JsonLdHelper
 
   # Returns account that owns the key
-  def call(uri, prefetched_json = nil)
-    @json = body_to_json(prefetched_json) || fetch_resource(uri)
+  def call(uri, id: true, prefetched_body: nil)
+    if prefetched_body.nil?
+      if id
+        @json = fetch_resource_without_id_validation(uri)
+        if person?
+          @json = fetch_resource(@json['id'], true)
+        elsif uri != @json['id']
+          return
+        end
+      else
+        @json = fetch_resource(uri, id)
+      end
+    else
+      @json = body_to_json(prefetched_body)
+    end
 
     return unless supported_context?(@json) && expected_type?
-    return find_account(uri, @json) if person?
+    return find_account(@json['id'], @json) if person?
 
-    @owner = fetch_resource(owner_uri)
+    @owner = fetch_resource(owner_uri, true)
 
     return unless supported_context?(@owner) && confirmed_owner?
 
@@ -19,9 +32,9 @@ class ActivityPub::FetchRemoteKeyService < BaseService
 
   private
 
-  def find_account(uri, prefetched_json)
+  def find_account(uri, prefetched_body)
     account   = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
-    account ||= ActivityPub::FetchRemoteAccountService.new.call(uri, prefetched_json)
+    account ||= ActivityPub::FetchRemoteAccountService.new.call(uri, prefetched_body: prefetched_body)
     account
   end
 
diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index a95931afe..c7414f161 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -4,36 +4,33 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   include JsonLdHelper
 
   # Should be called when uri has already been checked for locality
-  def call(uri, prefetched_json = nil)
-    @json = body_to_json(prefetched_json) || fetch_resource(uri)
+  def call(uri, id: true, prefetched_body: nil)
+    @json = if prefetched_body.nil?
+              fetch_resource(uri, id)
+            else
+              body_to_json(prefetched_body)
+            end
 
-    return unless supported_context?
+    return unless expected_type? && supported_context?
 
-    activity = activity_json
-    actor_id = value_or_id(activity['actor'])
-
-    return unless expected_type?(activity) && trustworthy_attribution?(uri, actor_id)
+    return if actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id)
 
     actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account)
-    actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id) if actor.nil?
+    actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil?
 
     return if actor.suspended?
 
-    ActivityPub::Activity.factory(activity, actor).perform
+    ActivityPub::Activity.factory(activity_json, actor).perform
   end
 
   private
 
   def activity_json
-    if %w(Note Article).include? @json['type']
-      {
-        'type'   => 'Create',
-        'actor'  => first_of_value(@json['attributedTo']),
-        'object' => @json,
-      }
-    else
-      @json
-    end
+    { 'type' => 'Create', 'actor' => actor_id, 'object' => @json }
+  end
+
+  def actor_id
+    first_of_value(@json['attributedTo'])
   end
 
   def trustworthy_attribution?(uri, attributed_to)
@@ -44,7 +41,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     super(@json)
   end
 
-  def expected_type?(json)
-    %w(Create Announce).include? json['type']
+  def expected_type?
+    %w(Note Article).include? @json['type']
   end
 end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 811209537..f93baf4b5 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -90,7 +90,7 @@ class ActivityPub::ProcessAccountService < BaseService
     return if value.nil?
     return value['url'] if value.is_a?(Hash)
 
-    image = fetch_resource(value)
+    image = fetch_resource_without_id_validation(value)
     image['url'] if image
   end
 
@@ -100,7 +100,7 @@ class ActivityPub::ProcessAccountService < BaseService
     return if value.nil?
     return value['publicKeyPem'] if value.is_a?(Hash)
 
-    key = fetch_resource(value)
+    key = fetch_resource_without_id_validation(value)
     key['publicKeyPem'] if key
   end
 
@@ -130,7 +130,7 @@ class ActivityPub::ProcessAccountService < BaseService
     return if @json[type].blank?
     return @collections[type] if @collections.key?(type)
 
-    collection = fetch_resource(@json[type])
+    collection = fetch_resource_without_id_validation(@json[type])
 
     @collections[type] = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
   rescue HTTP::Error, OpenSSL::SSL::SSLError
diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb
index 9c5777b5d..bcf516bc3 100644
--- a/app/services/fetch_atom_service.rb
+++ b/app/services/fetch_atom_service.rb
@@ -41,10 +41,11 @@ class FetchAtomService < BaseService
     return nil if @response.code != 200
 
     if @response.mime_type == 'application/atom+xml'
-      [@url, @response.to_s, :ostatus]
+      [@url, { prefetched_body: @response.to_s }, :ostatus]
     elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@response.mime_type)
-      if supported_activity?(@response.to_s)
-        [@url, @response.to_s, :activitypub]
+      json = body_to_json(body)
+      if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present?
+        [json['id'], { id: true }, :activitypub]
       else
         @unsupported_activity = true
         nil
@@ -79,10 +80,4 @@ class FetchAtomService < BaseService
 
     result
   end
-
-  def supported_activity?(body)
-    json = body_to_json(body)
-    return false unless supported_context?(json)
-    json['type'] == 'Person' ? json['inbox'].present? : true
-  end
 end
diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb
index bd98e70d1..a0f031a44 100644
--- a/app/services/fetch_remote_account_service.rb
+++ b/app/services/fetch_remote_account_service.rb
@@ -5,24 +5,24 @@ class FetchRemoteAccountService < BaseService
 
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
-      resource_url, body, protocol = FetchAtomService.new.call(url)
+      resource_url, resource_options, protocol = FetchAtomService.new.call(url)
     else
-      resource_url = url
-      body         = prefetched_body
+      resource_url     = url
+      resource_options = { prefetched_body: prefetched_body }
     end
 
     case protocol
     when :ostatus
-      process_atom(resource_url, body)
+      process_atom(resource_url, **resource_options)
     when :activitypub
-      ActivityPub::FetchRemoteAccountService.new.call(resource_url, body)
+      ActivityPub::FetchRemoteAccountService.new.call(resource_url, **resource_options)
     end
   end
 
   private
 
-  def process_atom(url, body)
-    xml = Nokogiri::XML(body)
+  def process_atom(url, prefetched_body:)
+    xml = Nokogiri::XML(prefetched_body)
     xml.encoding = 'utf-8'
 
     account = author_from_xml(xml.at_xpath('/xmlns:feed', xmlns: OStatus::TagManager::XMLNS), false)
diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index 1b90854c4..cacf6ba51 100644
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -5,26 +5,26 @@ class FetchRemoteStatusService < BaseService
 
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
-      resource_url, body, protocol = FetchAtomService.new.call(url)
+      resource_url, resource_options, protocol = FetchAtomService.new.call(url)
     else
-      resource_url = url
-      body         = prefetched_body
+      resource_url     = url
+      resource_options = { prefetched_body: prefetched_body }
     end
 
     case protocol
     when :ostatus
-      process_atom(resource_url, body)
+      process_atom(resource_url, **resource_options)
     when :activitypub
-      ActivityPub::FetchRemoteStatusService.new.call(resource_url, body)
+      ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options)
     end
   end
 
   private
 
-  def process_atom(url, body)
+  def process_atom(url, prefetched_body:)
     Rails.logger.debug "Processing Atom for remote status at #{url}"
 
-    xml = Nokogiri::XML(body)
+    xml = Nokogiri::XML(prefetched_body)
     xml.encoding = 'utf-8'
 
     account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
@@ -32,7 +32,7 @@ class FetchRemoteStatusService < BaseService
 
     return nil unless !account.nil? && confirmed_domain?(domain, account)
 
-    statuses = ProcessFeedService.new.call(body, account)
+    statuses = ProcessFeedService.new.call(prefetched_body, account)
     statuses.first
   rescue Nokogiri::XML::XPath::SyntaxError
     Rails.logger.debug 'Invalid XML or missing namespace'
diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb
index 93ba07702..3d0a36f6c 100644
--- a/app/services/resolve_remote_account_service.rb
+++ b/app/services/resolve_remote_account_service.rb
@@ -189,7 +189,7 @@ class ResolveRemoteAccountService < BaseService
   def actor_json
     return @actor_json if defined?(@actor_json)
 
-    json        = fetch_resource(actor_url)
+    json        = fetch_resource(actor_url, false)
     @actor_json = supported_context?(json) && json['type'] == 'Person' ? json : nil
   end
 
diff --git a/spec/helpers/jsonld_helper_spec.rb b/spec/helpers/jsonld_helper_spec.rb
index 7d3912e6c..48bfdc306 100644
--- a/spec/helpers/jsonld_helper_spec.rb
+++ b/spec/helpers/jsonld_helper_spec.rb
@@ -30,6 +30,39 @@ describe JsonLdHelper do
   end
 
   describe '#fetch_resource' do
-    pending
+    context 'when the second argument is false' do
+      it 'returns resource even if the retrieved ID and the given URI does not match' do
+        stub_request(:get, 'https://bob/').to_return body: '{"id": "https://alice/"}'
+        stub_request(:get, 'https://alice/').to_return body: '{"id": "https://alice/"}'
+
+        expect(fetch_resource('https://bob/', false)).to eq({ 'id' => 'https://alice/' })
+      end
+
+      it 'returns nil if the object identified by the given URI and the object identified by the retrieved ID does not match' do
+        stub_request(:get, 'https://mallory/').to_return body: '{"id": "https://marvin/"}'
+        stub_request(:get, 'https://marvin/').to_return body: '{"id": "https://alice/"}'
+
+        expect(fetch_resource('https://mallory/', false)).to eq nil
+      end
+    end
+
+    context 'when the second argument is true' do
+      it 'returns nil if the retrieved ID and the given URI does not match' do
+        stub_request(:get, 'https://mallory/').to_return body: '{"id": "https://alice/"}'
+        expect(fetch_resource('https://mallory/', true)).to eq nil
+      end
+    end
+  end
+
+  describe '#fetch_resource_without_id_validation' do
+    it 'returns nil if the status code is not 200' do
+      stub_request(:get, 'https://host/').to_return status: 400, body: '{}'
+      expect(fetch_resource_without_id_validation('https://host/')).to eq nil
+    end
+
+    it 'returns hash' do
+      stub_request(:get, 'https://host/').to_return status: 200, body: '{}'
+      expect(fetch_resource_without_id_validation('https://host/')).to eq({})
+    end
   end
 end
diff --git a/spec/services/activitypub/fetch_remote_account_service_spec.rb b/spec/services/activitypub/fetch_remote_account_service_spec.rb
index ed7e9bba8..c50d3fb97 100644
--- a/spec/services/activitypub/fetch_remote_account_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_account_service_spec.rb
@@ -16,7 +16,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService do
   end
 
   describe '#call' do
-    let(:account) { subject.call('https://example.com/alice') }
+    let(:account) { subject.call('https://example.com/alice', id: true) }
 
     shared_examples 'sets profile data' do
       it 'returns an account' do
diff --git a/spec/services/activitypub/fetch_remote_status_service_spec.rb b/spec/services/activitypub/fetch_remote_status_service_spec.rb
index 3b22257ed..ebf422392 100644
--- a/spec/services/activitypub/fetch_remote_status_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_status_service_spec.rb
@@ -15,21 +15,11 @@ RSpec.describe ActivityPub::FetchRemoteStatusService do
     }
   end
 
-  let(:create) do
-    {
-      '@context': 'https://www.w3.org/ns/activitystreams',
-      id: "https://#{valid_domain}/@foo/1234/activity",
-      type: 'Create',
-      actor: ActivityPub::TagManager.instance.uri_for(sender),
-      object: note,
-    }
-  end
-
   subject { described_class.new }
 
   describe '#call' do
     before do
-      subject.call(object[:id], Oj.dump(object))
+      subject.call(object[:id], prefetched_body: Oj.dump(object))
     end
 
     context 'with Note object' do
@@ -42,34 +32,5 @@ RSpec.describe ActivityPub::FetchRemoteStatusService do
         expect(status.text).to eq 'Lorem ipsum'
       end
     end
-
-    context 'with Create activity' do
-      let(:object) { create }
-
-      it 'creates status' do
-        status = sender.statuses.first
-        
-        expect(status).to_not be_nil
-        expect(status.text).to eq 'Lorem ipsum'
-      end
-    end
-
-    context 'with Announce activity' do
-      let(:status) { Fabricate(:status, account: recipient) }
-
-      let(:object) do
-        {
-          '@context': 'https://www.w3.org/ns/activitystreams',
-          id: "https://#{valid_domain}/@foo/1234/activity",
-          type: 'Announce',
-          actor: ActivityPub::TagManager.instance.uri_for(sender),
-          object: ActivityPub::TagManager.instance.uri_for(status),
-        }
-      end
-
-      it 'creates a reblog by sender of status' do
-        expect(sender.reblogged?(status)).to be true
-      end
-    end
   end
 end
-- 
cgit 


From 3a3475450e46f670e8beaf4bf804b820ad39a5f9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 7 Oct 2017 17:43:42 +0200
Subject: Encode custom emojis as resolveable objects in ActivityPub (#5243)

* Encode custom emojis as resolveable objects in ActivityPub

* Improve code style
---
 app/controllers/accounts_controller.rb             |  5 +++-
 app/controllers/emojis_controller.rb               | 22 ++++++++++++++++
 app/controllers/follower_accounts_controller.rb    |  5 +++-
 app/controllers/following_accounts_controller.rb   |  5 +++-
 app/controllers/statuses_controller.rb             | 10 ++++++--
 app/controllers/tags_controller.rb                 |  5 +++-
 app/lib/activitypub/activity/create.rb             | 12 ++++++---
 app/lib/activitypub/tag_manager.rb                 |  2 ++
 app/models/custom_emoji.rb                         |  6 +++++
 app/serializers/activitypub/actor_serializer.rb    | 18 ++------------
 app/serializers/activitypub/emoji_serializer.rb    | 29 ++++++++++++++++++++++
 app/serializers/activitypub/image_serializer.rb    | 19 ++++++++++++++
 app/serializers/activitypub/note_serializer.rb     | 17 +------------
 config/routes.rb                                   |  5 ++--
 .../20171006142024_add_uri_to_custom_emojis.rb     |  6 +++++
 db/schema.rb                                       |  4 ++-
 spec/lib/activitypub/activity/create_spec.rb       | 10 +++++---
 17 files changed, 132 insertions(+), 48 deletions(-)
 create mode 100644 app/controllers/emojis_controller.rb
 create mode 100644 app/serializers/activitypub/emoji_serializer.rb
 create mode 100644 app/serializers/activitypub/image_serializer.rb
 create mode 100644 db/migrate/20171006142024_add_uri_to_custom_emojis.rb

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 26ab6636b..75915b337 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -26,7 +26,10 @@ class AccountsController < ApplicationController
       end
 
       format.json do
-        render json: @account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+        render json: @account,
+               serializer: ActivityPub::ActorSerializer,
+               adapter: ActivityPub::Adapter,
+               content_type: 'application/activity+json'
       end
     end
   end
diff --git a/app/controllers/emojis_controller.rb b/app/controllers/emojis_controller.rb
new file mode 100644
index 000000000..a82b9340b
--- /dev/null
+++ b/app/controllers/emojis_controller.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class EmojisController < ApplicationController
+  before_action :set_emoji
+
+  def show
+    respond_to do |format|
+      format.json do
+        render json: @emoji,
+               serializer: ActivityPub::EmojiSerializer,
+               adapter: ActivityPub::Adapter,
+               content_type: 'application/activity+json'
+      end
+    end
+  end
+
+  private
+
+  def set_emoji
+    @emoji = CustomEmoji.local.find(params[:id])
+  end
+end
diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb
index 8eb4d2822..399e79665 100644
--- a/app/controllers/follower_accounts_controller.rb
+++ b/app/controllers/follower_accounts_controller.rb
@@ -10,7 +10,10 @@ class FollowerAccountsController < ApplicationController
       format.html
 
       format.json do
-        render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+        render json: collection_presenter,
+               serializer: ActivityPub::CollectionSerializer,
+               adapter: ActivityPub::Adapter,
+               content_type: 'application/activity+json'
       end
     end
   end
diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb
index 1ca6f0fe7..1e73d4bd4 100644
--- a/app/controllers/following_accounts_controller.rb
+++ b/app/controllers/following_accounts_controller.rb
@@ -10,7 +10,10 @@ class FollowingAccountsController < ApplicationController
       format.html
 
       format.json do
-        render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+        render json: collection_presenter,
+               serializer: ActivityPub::CollectionSerializer,
+               adapter: ActivityPub::Adapter,
+               content_type: 'application/activity+json'
       end
     end
   end
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 65206ea96..e8a360fb5 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -21,13 +21,19 @@ class StatusesController < ApplicationController
       end
 
       format.json do
-        render json: @status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+        render json: @status,
+               serializer: ActivityPub::NoteSerializer,
+               adapter: ActivityPub::Adapter,
+               content_type: 'application/activity+json'
       end
     end
   end
 
   def activity
-    render json: @status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+    render json: @status,
+           serializer: ActivityPub::ActivitySerializer,
+           adapter: ActivityPub::Adapter,
+           content_type: 'application/activity+json'
   end
 
   def embed
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index 3001b2ee3..240ef058a 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -12,7 +12,10 @@ class TagsController < ApplicationController
       format.html
 
       format.json do
-        render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+        render json: collection_presenter,
+               serializer: ActivityPub::CollectionSerializer,
+               adapter: ActivityPub::Adapter,
+               content_type: 'application/activity+json'
       end
     end
   end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index be656de48..9421a0aa7 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -86,15 +86,19 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_emoji(tag, _status)
-    return if tag['name'].blank? || tag['href'].blank?
+    return if skip_download?
+    return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
 
     shortcode = tag['name'].delete(':')
+    image_url = tag['icon']['url']
+    uri       = tag['id']
+    updated   = tag['updated']
     emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
 
-    return if !emoji.nil? || skip_download?
+    return unless emoji.nil? || emoji.updated_at >= updated
 
-    emoji = CustomEmoji.new(domain: @account.domain, shortcode: shortcode)
-    emoji.image_remote_url = tag['href']
+    emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
+    emoji.image_remote_url = image_url
     emoji.save
   end
 
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 4ec3b8c56..0708713e6 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -33,6 +33,8 @@ class ActivityPub::TagManager
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       account_status_url(target.account, target)
+    when :emoji
+      emoji_url(target)
     end
   end
 
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 258b50c82..65d9840d5 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -13,6 +13,8 @@
 #  created_at         :datetime         not null
 #  updated_at         :datetime         not null
 #  disabled           :boolean          default(FALSE), not null
+#  uri                :string
+#  image_remote_url   :string
 #
 
 class CustomEmoji < ApplicationRecord
@@ -37,6 +39,10 @@ class CustomEmoji < ApplicationRecord
     domain.nil?
   end
 
+  def object_type
+    :emoji
+  end
+
   class << self
     def from_text(text, domain)
       return [] if text.blank?
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index a11178f5b..896d67115 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -10,20 +10,6 @@ class ActivityPub::ActorSerializer < ActiveModel::Serializer
 
   has_one :public_key, serializer: ActivityPub::PublicKeySerializer
 
-  class ImageSerializer < ActiveModel::Serializer
-    include RoutingHelper
-
-    attributes :type, :url
-
-    def type
-      'Image'
-    end
-
-    def url
-      full_asset_url(object.url(:original))
-    end
-  end
-
   class EndpointsSerializer < ActiveModel::Serializer
     include RoutingHelper
 
@@ -36,8 +22,8 @@ class ActivityPub::ActorSerializer < ActiveModel::Serializer
 
   has_one :endpoints, serializer: EndpointsSerializer
 
-  has_one :icon,  serializer: ImageSerializer, if: :avatar_exists?
-  has_one :image, serializer: ImageSerializer, if: :header_exists?
+  has_one :icon,  serializer: ActivityPub::ImageSerializer, if: :avatar_exists?
+  has_one :image, serializer: ActivityPub::ImageSerializer, if: :header_exists?
 
   def id
     account_url(object)
diff --git a/app/serializers/activitypub/emoji_serializer.rb b/app/serializers/activitypub/emoji_serializer.rb
new file mode 100644
index 000000000..7b06b1e5d
--- /dev/null
+++ b/app/serializers/activitypub/emoji_serializer.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class ActivityPub::EmojiSerializer < ActiveModel::Serializer
+  include RoutingHelper
+
+  attributes :id, :type, :name, :updated
+
+  has_one :icon, serializer: ActivityPub::ImageSerializer
+
+  def id
+    ActivityPub::TagManager.instance.uri_for(object)
+  end
+
+  def type
+    'Emoji'
+  end
+
+  def icon
+    object.image
+  end
+
+  def updated
+    object.updated_at.iso8601
+  end
+
+  def name
+    ":#{object.shortcode}:"
+  end
+end
diff --git a/app/serializers/activitypub/image_serializer.rb b/app/serializers/activitypub/image_serializer.rb
new file mode 100644
index 000000000..a015c6b1b
--- /dev/null
+++ b/app/serializers/activitypub/image_serializer.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class ActivityPub::ImageSerializer < ActiveModel::Serializer
+  include RoutingHelper
+
+  attributes :type, :media_type, :url
+
+  def type
+    'Image'
+  end
+
+  def url
+    full_asset_url(object.url(:original))
+  end
+
+  def media_type
+    object.content_type
+  end
+end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 4dbf6a444..24c39f3c9 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -142,21 +142,6 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
     end
   end
 
-  class CustomEmojiSerializer < ActiveModel::Serializer
-    include RoutingHelper
-
-    attributes :type, :href, :name
-
-    def type
-      'Emoji'
-    end
-
-    def href
-      full_asset_url(object.image.url)
-    end
-
-    def name
-      ":#{object.shortcode}:"
-    end
+  class CustomEmojiSerializer < ActivityPub::EmojiSerializer
   end
 end
diff --git a/config/routes.rb b/config/routes.rb
index cc1f66e52..bd7068b5c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -96,8 +96,9 @@ Rails.application.routes.draw do
     resources :sessions, only: [:destroy]
   end
 
-  resources :media, only: [:show]
-  resources :tags,  only: [:show]
+  resources :media,  only: [:show]
+  resources :tags,   only: [:show]
+  resources :emojis, only: [:show]
 
   get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy
 
diff --git a/db/migrate/20171006142024_add_uri_to_custom_emojis.rb b/db/migrate/20171006142024_add_uri_to_custom_emojis.rb
new file mode 100644
index 000000000..04dfcf397
--- /dev/null
+++ b/db/migrate/20171006142024_add_uri_to_custom_emojis.rb
@@ -0,0 +1,6 @@
+class AddUriToCustomEmojis < ActiveRecord::Migration[5.1]
+  def change
+    add_column :custom_emojis, :uri, :string
+    add_column :custom_emojis, :image_remote_url, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3358e2997..7180d3515 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171005171936) do
+ActiveRecord::Schema.define(version: 20171006142024) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -99,6 +99,8 @@ ActiveRecord::Schema.define(version: 20171005171936) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.boolean "disabled", default: false, null: false
+    t.string "uri"
+    t.string "image_remote_url"
     t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
   end
 
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index cdd499150..3c3991c13 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -290,7 +290,9 @@ RSpec.describe ActivityPub::Activity::Create do
           tag: [
             {
               type: 'Emoji',
-              href: 'http://example.com/emoji.png',
+              icon: {
+                url: 'http://example.com/emoji.png',
+              },
               name: 'tinking',
             },
           ],
@@ -314,7 +316,9 @@ RSpec.describe ActivityPub::Activity::Create do
           tag: [
             {
               type: 'Emoji',
-              href: 'http://example.com/emoji.png',
+              icon: {
+                url: 'http://example.com/emoji.png',
+              },
             },
           ],
         }
@@ -326,7 +330,7 @@ RSpec.describe ActivityPub::Activity::Create do
       end
     end
 
-    context 'with emojis missing href' do
+    context 'with emojis missing icon' do
       let(:object_json) do
         {
           id: 'bar',
-- 
cgit 


From 0717d9b3e6904a4dcd5d2dc9e680cc5b21c50e51 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 8 Oct 2017 17:34:34 +0200
Subject: Set snowflake IDs for backdated statuses (#5260)

- Rename Mastodon::TimestampIds into Mastodon::Snowflake for clarity
- Skip for statuses coming from inbox, aka delivered in real-time
- Skip for statuses that claim to be from the future
---
 app/lib/activitypub/activity.rb                    |   7 +-
 app/lib/activitypub/activity/announce.rb           |   3 +-
 app/lib/activitypub/activity/create.rb             |   2 +-
 app/lib/ostatus/activity/base.rb                   |   5 +-
 app/lib/ostatus/activity/creation.rb               |   2 +-
 app/lib/ostatus/activity/general.rb                |   2 +-
 app/models/status.rb                               |   2 +
 .../activitypub/process_collection_service.rb      |   5 +-
 app/services/process_feed_service.rb               |   6 +-
 app/workers/activitypub/processing_worker.rb       |   2 +-
 app/workers/processing_worker.rb                   |   2 +-
 config/application.rb                              |   1 +
 config/brakeman.ignore                             |  44 +++---
 lib/mastodon/snowflake.rb                          | 162 +++++++++++++++++++++
 lib/mastodon/timestamp_ids.rb                      | 131 -----------------
 lib/tasks/db.rake                                  |   6 +-
 .../activitypub/process_collection_service_spec.rb |   4 +-
 17 files changed, 213 insertions(+), 173 deletions(-)
 create mode 100644 lib/mastodon/snowflake.rb
 delete mode 100644 lib/mastodon/timestamp_ids.rb

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index b06dd6194..9688f57a6 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -3,10 +3,11 @@
 class ActivityPub::Activity
   include JsonLdHelper
 
-  def initialize(json, account)
+  def initialize(json, account, options = {})
     @json    = json
     @account = account
     @object  = @json['object']
+    @options = options
   end
 
   def perform
@@ -14,9 +15,9 @@ class ActivityPub::Activity
   end
 
   class << self
-    def factory(json, account)
+    def factory(json, account, options = {})
       @json = json
-      klass&.new(json, account)
+      klass&.new(json, account, options)
     end
 
     private
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 1cf844281..b84098933 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -15,8 +15,9 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       account: @account,
       reblog: original_status,
       uri: @json['id'],
-      created_at: @json['published'] || Time.now.utc
+      created_at: @options[:override_timestamps] ? nil : @json['published']
     )
+
     distribute(status)
     status
   end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 9421a0aa7..d6e9bc1de 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -43,7 +43,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       text: text_from_content || '',
       language: language_from_content,
       spoiler_text: @object['summary'] || '',
-      created_at: @object['published'] || Time.now.utc,
+      created_at: @options[:override_timestamps] ? nil : @object['published'],
       reply: @object['inReplyTo'].present?,
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,
diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb
index 039381397..8b27b124f 100644
--- a/app/lib/ostatus/activity/base.rb
+++ b/app/lib/ostatus/activity/base.rb
@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 class OStatus::Activity::Base
-  def initialize(xml, account = nil)
-    @xml = xml
+  def initialize(xml, account = nil, options = {})
+    @xml     = xml
     @account = account
+    @options = options
   end
 
   def status?
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 511c462d4..a1ab522e2 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -34,7 +34,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
         reblog: cached_reblog,
         text: content,
         spoiler_text: content_warning,
-        created_at: published,
+        created_at: @options[:override_timestamps] ? nil : published,
         reply: thread?,
         language: content_language,
         visibility: visibility_scope,
diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb
index b3bef9861..8a6aabc33 100644
--- a/app/lib/ostatus/activity/general.rb
+++ b/app/lib/ostatus/activity/general.rb
@@ -2,7 +2,7 @@
 
 class OStatus::Activity::General < OStatus::Activity::Base
   def specialize
-    special_class&.new(@xml, @account)
+    special_class&.new(@xml, @account, @options)
   end
 
   private
diff --git a/app/models/status.rb b/app/models/status.rb
index ea4c097bf..0d249244f 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -136,6 +136,8 @@ class Status < ApplicationRecord
 
   after_create :store_uri, if: :local?
 
+  around_create Mastodon::Snowflake::Callbacks
+
   before_validation :prepare_contents, if: :local?
   before_validation :set_reblog
   before_validation :set_visibility
diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb
index 59cb65c65..db4d1b4bc 100644
--- a/app/services/activitypub/process_collection_service.rb
+++ b/app/services/activitypub/process_collection_service.rb
@@ -3,9 +3,10 @@
 class ActivityPub::ProcessCollectionService < BaseService
   include JsonLdHelper
 
-  def call(body, account)
+  def call(body, account, options = {})
     @account = account
     @json    = Oj.load(body, mode: :strict)
+    @options = options
 
     return unless supported_context?
     return if different_actor? && verify_account!.nil?
@@ -38,7 +39,7 @@ class ActivityPub::ProcessCollectionService < BaseService
   end
 
   def process_item(item)
-    activity = ActivityPub::Activity.factory(item, @account)
+    activity = ActivityPub::Activity.factory(item, @account, @options)
     activity&.perform
   end
 
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
index 2a5f1e2bc..60eff135e 100644
--- a/app/services/process_feed_service.rb
+++ b/app/services/process_feed_service.rb
@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 class ProcessFeedService < BaseService
-  def call(body, account)
+  def call(body, account, options = {})
+    @options = options
+
     xml = Nokogiri::XML(body)
     xml.encoding = 'utf-8'
 
@@ -20,7 +22,7 @@ class ProcessFeedService < BaseService
   end
 
   def process_entry(xml, account)
-    activity = OStatus::Activity::General.new(xml, account)
+    activity = OStatus::Activity::General.new(xml, account, @options)
     activity.specialize&.perform if activity.status?
   rescue ActiveRecord::RecordInvalid => e
     Rails.logger.debug "Nothing was saved for #{activity.id} because: #{e}"
diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb
index bb9adf64b..0e2e0eddd 100644
--- a/app/workers/activitypub/processing_worker.rb
+++ b/app/workers/activitypub/processing_worker.rb
@@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id))
+    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc..978c3aba2 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -6,6 +6,6 @@ class ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id))
+    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end
diff --git a/config/application.rb b/config/application.rb
index b6ce74147..4860a08a1 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -9,6 +9,7 @@ Bundler.require(*Rails.groups)
 require_relative '../app/lib/exceptions'
 require_relative '../lib/paperclip/gif_transcoder'
 require_relative '../lib/paperclip/video_transcoder'
+require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
 
 Dotenv::Railtie.load
diff --git a/config/brakeman.ignore b/config/brakeman.ignore
index 2a1bc1997..f198eebac 100644
--- a/config/brakeman.ignore
+++ b/config/brakeman.ignore
@@ -57,26 +57,6 @@
       "confidence": "Weak",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "34efc76883080f8b1110a30c34ec4f903946ee56651aae46c62477f45d4fc412",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "lib/mastodon/timestamp_ids.rb",
-      "line": 63,
-      "link": "http://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "connection.execute(\"        CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n        RETURNS bigint AS\\n        $$\\n          DECLARE\\n            time_part bigint;\\n            sequence_base bigint;\\n            tail bigint;\\n          BEGIN\\n            time_part := (\\n              -- Get the time in milliseconds\\n              ((date_part('epoch', now()) * 1000))::bigint\\n              -- And shift it over two bytes\\n              << 16);\\n\\n            sequence_base := (\\n              'x' ||\\n              -- Take the first two bytes (four hex characters)\\n              substr(\\n                -- Of the MD5 hash of the data we documented\\n                md5(table_name ||\\n                  '#{SecureRandom.hex(16)}' ||\\n                  time_part::text\\n                ),\\n                1, 4\\n              )\\n            -- And turn it into a bigint\\n            )::bit(16)::bigint;\\n\\n            -- Finally, add our sequence number to our base, and chop\\n            -- it to the last two bytes\\n            tail := (\\n              (sequence_base + nextval(table_name || '_id_seq'))\\n              & 65535);\\n\\n            -- Return the time part and the sequence part. OR appears\\n            -- faster here than addition, but they're equivalent:\\n            -- time_part has no trailing two bytes, and tail is only\\n            -- the last two bytes.\\n            RETURN time_part | tail;\\n          END\\n        $$ LANGUAGE plpgsql VOLATILE;\\n\")",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Mastodon::TimestampIds",
-        "method": "define_timestamp_id"
-      },
-      "user_input": "SecureRandom.hex(16)",
-      "confidence": "Medium",
-      "note": ""
-    },
     {
       "warning_type": "Dynamic Render Path",
       "warning_code": 15,
@@ -106,7 +86,7 @@
       "line": 3,
       "link": "http://brakemanscanner.org/docs/warning_types/dynamic_render_path/",
       "code": "render(action => \"stream_entries/#{Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity_type.downcase}\", { Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity_type.downcase.to_sym => Account.find_local!(params[:account_username]).statuses.find(params[:id]).stream_entry.activity, :centered => true })",
-      "render_path": [{"type":"controller","class":"StatusesController","method":"embed","line":35,"file":"app/controllers/statuses_controller.rb"}],
+      "render_path": [{"type":"controller","class":"StatusesController","method":"embed","line":41,"file":"app/controllers/statuses_controller.rb"}],
       "location": {
         "type": "template",
         "template": "stream_entries/embed"
@@ -153,6 +133,26 @@
       "confidence": "Weak",
       "note": ""
     },
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "lib/mastodon/snowflake.rb",
+      "line": 86,
+      "link": "http://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "connection.execute(\"        CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n        RETURNS bigint AS\\n        $$\\n          DECLARE\\n            time_part bigint;\\n            sequence_base bigint;\\n            tail bigint;\\n          BEGIN\\n            time_part := (\\n              -- Get the time in milliseconds\\n              ((date_part('epoch', now()) * 1000))::bigint\\n              -- And shift it over two bytes\\n              << 16);\\n\\n            sequence_base := (\\n              'x' ||\\n              -- Take the first two bytes (four hex characters)\\n              substr(\\n                -- Of the MD5 hash of the data we documented\\n                md5(table_name ||\\n                  '#{SecureRandom.hex(16)}' ||\\n                  time_part::text\\n                ),\\n                1, 4\\n              )\\n            -- And turn it into a bigint\\n            )::bit(16)::bigint;\\n\\n            -- Finally, add our sequence number to our base, and chop\\n            -- it to the last two bytes\\n            tail := (\\n              (sequence_base + nextval(table_name || '_id_seq'))\\n              & 65535);\\n\\n            -- Return the time part and the sequence part. OR appears\\n            -- faster here than addition, but they're equivalent:\\n            -- time_part has no trailing two bytes, and tail is only\\n            -- the last two bytes.\\n            RETURN time_part | tail;\\n          END\\n        $$ LANGUAGE plpgsql VOLATILE;\\n\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "Mastodon::Snowflake",
+        "method": "define_timestamp_id"
+      },
+      "user_input": "SecureRandom.hex(16)",
+      "confidence": "Medium",
+      "note": ""
+    },
     {
       "warning_type": "Dynamic Render Path",
       "warning_code": 15,
@@ -269,6 +269,6 @@
       "note": ""
     }
   ],
-  "updated": "2017-10-06 03:27:46 +0200",
+  "updated": "2017-10-07 19:24:02 +0200",
   "brakeman_version": "4.0.1"
 }
diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb
new file mode 100644
index 000000000..219e323d4
--- /dev/null
+++ b/lib/mastodon/snowflake.rb
@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Mastodon::Snowflake
+  DEFAULT_REGEX = /timestamp_id\('(?<seq_prefix>\w+)'/
+
+  class Callbacks
+    def self.around_create(record)
+      now = Time.now.utc
+
+      if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at
+        yield
+      else
+        record.id = Mastodon::Snowflake.id_at(record.created_at)
+        tries     = 0
+
+        begin
+          yield
+        rescue ActiveRecord::RecordNotUnique
+          raise if tries > 100
+
+          tries     += 1
+          record.id += rand(100)
+
+          retry
+        end
+      end
+    end
+  end
+
+  class << self
+    # Our ID will be composed of the following:
+    # 6 bytes (48 bits) of millisecond-level timestamp
+    # 2 bytes (16 bits) of sequence data
+    #
+    # The 'sequence data' is intended to be unique within a
+    # given millisecond, yet obscure the 'serial number' of
+    # this row.
+    #
+    # To do this, we hash the following data:
+    # * Table name (if provided, skipped if not)
+    # * Secret salt (should not be guessable)
+    # * Timestamp (again, millisecond-level granularity)
+    #
+    # We then take the first two bytes of that value, and add
+    # the lowest two bytes of the table ID sequence number
+    # (`table_name`_id_seq). This means that even if we insert
+    # two rows at the same millisecond, they will have
+    # distinct 'sequence data' portions.
+    #
+    # If this happens, and an attacker can see both such IDs,
+    # they can determine which of the two entries was inserted
+    # first, but not the total number of entries in the table
+    # (even mod 2**16).
+    #
+    # The table name is included in the hash to ensure that
+    # different tables derive separate sequence bases so rows
+    # inserted in the same millisecond in different tables do
+    # not reveal the table ID sequence number for one another.
+    #
+    # The secret salt is included in the hash to ensure that
+    # external users cannot derive the sequence base given the
+    # timestamp and table name, which would allow them to
+    # compute the table ID sequence number.
+    def define_timestamp_id
+      return if already_defined?
+
+      connection.execute(<<~SQL)
+        CREATE OR REPLACE FUNCTION timestamp_id(table_name text)
+        RETURNS bigint AS
+        $$
+          DECLARE
+            time_part bigint;
+            sequence_base bigint;
+            tail bigint;
+          BEGIN
+            time_part := (
+              -- Get the time in milliseconds
+              ((date_part('epoch', now()) * 1000))::bigint
+              -- And shift it over two bytes
+              << 16);
+
+            sequence_base := (
+              'x' ||
+              -- Take the first two bytes (four hex characters)
+              substr(
+                -- Of the MD5 hash of the data we documented
+                md5(table_name ||
+                  '#{SecureRandom.hex(16)}' ||
+                  time_part::text
+                ),
+                1, 4
+              )
+            -- And turn it into a bigint
+            )::bit(16)::bigint;
+
+            -- Finally, add our sequence number to our base, and chop
+            -- it to the last two bytes
+            tail := (
+              (sequence_base + nextval(table_name || '_id_seq'))
+              & 65535);
+
+            -- Return the time part and the sequence part. OR appears
+            -- faster here than addition, but they're equivalent:
+            -- time_part has no trailing two bytes, and tail is only
+            -- the last two bytes.
+            RETURN time_part | tail;
+          END
+        $$ LANGUAGE plpgsql VOLATILE;
+      SQL
+    end
+
+    def ensure_id_sequences_exist
+      # Find tables using timestamp IDs.
+      connection.tables.each do |table|
+        # We're only concerned with "id" columns.
+        next unless (id_col = connection.columns(table).find { |col| col.name == 'id' })
+
+        # And only those that are using timestamp_id.
+        next unless (data = DEFAULT_REGEX.match(id_col.default_function))
+
+        seq_name = data[:seq_prefix] + '_id_seq'
+
+        # If we were on Postgres 9.5+, we could do CREATE SEQUENCE IF
+        # NOT EXISTS, but we can't depend on that. Instead, catch the
+        # possible exception and ignore it.
+        # Note that seq_name isn't a column name, but it's a
+        # relation, like a column, and follows the same quoting rules
+        # in Postgres.
+        connection.execute(<<~SQL)
+          DO $$
+            BEGIN
+              CREATE SEQUENCE #{connection.quote_column_name(seq_name)};
+            EXCEPTION WHEN duplicate_table THEN
+              -- Do nothing, we have the sequence already.
+            END
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+
+    def id_at(timestamp)
+      id  = timestamp.to_i * 1000 + rand(1000)
+      id  = id << 16
+      id += rand(2**16)
+      id
+    end
+
+    private
+
+    def already_defined?
+      connection.execute(<<~SQL).values.first.first
+        SELECT EXISTS(
+          SELECT * FROM pg_proc WHERE proname = 'timestamp_id'
+        );
+      SQL
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end
diff --git a/lib/mastodon/timestamp_ids.rb b/lib/mastodon/timestamp_ids.rb
deleted file mode 100644
index 3b048a50c..000000000
--- a/lib/mastodon/timestamp_ids.rb
+++ /dev/null
@@ -1,131 +0,0 @@
-# frozen_string_literal: true
-
-module Mastodon::TimestampIds
-  DEFAULT_REGEX = /timestamp_id\('(?<seq_prefix>\w+)'/
-
-  class << self
-    # Our ID will be composed of the following:
-    # 6 bytes (48 bits) of millisecond-level timestamp
-    # 2 bytes (16 bits) of sequence data
-    #
-    # The 'sequence data' is intended to be unique within a
-    # given millisecond, yet obscure the 'serial number' of
-    # this row.
-    #
-    # To do this, we hash the following data:
-    # * Table name (if provided, skipped if not)
-    # * Secret salt (should not be guessable)
-    # * Timestamp (again, millisecond-level granularity)
-    #
-    # We then take the first two bytes of that value, and add
-    # the lowest two bytes of the table ID sequence number
-    # (`table_name`_id_seq). This means that even if we insert
-    # two rows at the same millisecond, they will have
-    # distinct 'sequence data' portions.
-    #
-    # If this happens, and an attacker can see both such IDs,
-    # they can determine which of the two entries was inserted
-    # first, but not the total number of entries in the table
-    # (even mod 2**16).
-    #
-    # The table name is included in the hash to ensure that
-    # different tables derive separate sequence bases so rows
-    # inserted in the same millisecond in different tables do
-    # not reveal the table ID sequence number for one another.
-    #
-    # The secret salt is included in the hash to ensure that
-    # external users cannot derive the sequence base given the
-    # timestamp and table name, which would allow them to
-    # compute the table ID sequence number.
-    def define_timestamp_id
-      return if already_defined?
-
-      connection.execute(<<~SQL)
-        CREATE OR REPLACE FUNCTION timestamp_id(table_name text)
-        RETURNS bigint AS
-        $$
-          DECLARE
-            time_part bigint;
-            sequence_base bigint;
-            tail bigint;
-          BEGIN
-            time_part := (
-              -- Get the time in milliseconds
-              ((date_part('epoch', now()) * 1000))::bigint
-              -- And shift it over two bytes
-              << 16);
-
-            sequence_base := (
-              'x' ||
-              -- Take the first two bytes (four hex characters)
-              substr(
-                -- Of the MD5 hash of the data we documented
-                md5(table_name ||
-                  '#{SecureRandom.hex(16)}' ||
-                  time_part::text
-                ),
-                1, 4
-              )
-            -- And turn it into a bigint
-            )::bit(16)::bigint;
-
-            -- Finally, add our sequence number to our base, and chop
-            -- it to the last two bytes
-            tail := (
-              (sequence_base + nextval(table_name || '_id_seq'))
-              & 65535);
-
-            -- Return the time part and the sequence part. OR appears
-            -- faster here than addition, but they're equivalent:
-            -- time_part has no trailing two bytes, and tail is only
-            -- the last two bytes.
-            RETURN time_part | tail;
-          END
-        $$ LANGUAGE plpgsql VOLATILE;
-      SQL
-    end
-
-    def ensure_id_sequences_exist
-      # Find tables using timestamp IDs.
-      connection.tables.each do |table|
-        # We're only concerned with "id" columns.
-        next unless (id_col = connection.columns(table).find { |col| col.name == 'id' })
-
-        # And only those that are using timestamp_id.
-        next unless (data = DEFAULT_REGEX.match(id_col.default_function))
-
-        seq_name = data[:seq_prefix] + '_id_seq'
-
-        # If we were on Postgres 9.5+, we could do CREATE SEQUENCE IF
-        # NOT EXISTS, but we can't depend on that. Instead, catch the
-        # possible exception and ignore it.
-        # Note that seq_name isn't a column name, but it's a
-        # relation, like a column, and follows the same quoting rules
-        # in Postgres.
-        connection.execute(<<~SQL)
-          DO $$
-            BEGIN
-              CREATE SEQUENCE #{connection.quote_column_name(seq_name)};
-            EXCEPTION WHEN duplicate_table THEN
-              -- Do nothing, we have the sequence already.
-            END
-          $$ LANGUAGE plpgsql;
-        SQL
-      end
-    end
-
-    private
-
-    def already_defined?
-      connection.execute(<<~SQL).values.first.first
-        SELECT EXISTS(
-          SELECT * FROM pg_proc WHERE proname = 'timestamp_id'
-        );
-      SQL
-    end
-
-    def connection
-      ActiveRecord::Base.connection
-    end
-  end
-end
diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index 6af6bb6fb..32039c31d 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require Rails.root.join('lib', 'mastodon', 'timestamp_ids')
+require_relative '../mastodon/snowflake'
 
 def each_schema_load_environment
   # If we're in development, also run this for the test environment.
@@ -63,13 +63,13 @@ namespace :db do
 
   task :define_timestamp_id do
     each_schema_load_environment do
-      Mastodon::TimestampIds.define_timestamp_id
+      Mastodon::Snowflake.define_timestamp_id
     end
   end
 
   task :ensure_id_sequences_exist do
     each_schema_load_environment do
-      Mastodon::TimestampIds.ensure_id_sequences_exist
+      Mastodon::Snowflake.ensure_id_sequences_exist
     end
   end
 end
diff --git a/spec/services/activitypub/process_collection_service_spec.rb b/spec/services/activitypub/process_collection_service_spec.rb
index c1cc22523..3cea970cf 100644
--- a/spec/services/activitypub/process_collection_service_spec.rb
+++ b/spec/services/activitypub/process_collection_service_spec.rb
@@ -28,7 +28,7 @@ RSpec.describe ActivityPub::ProcessCollectionService do
 
       it 'processes payload with sender if no signature exists' do
         expect_any_instance_of(ActivityPub::LinkedDataSignature).not_to receive(:verify_account!)
-        expect(ActivityPub::Activity).to receive(:factory).with(instance_of(Hash), forwarder)
+        expect(ActivityPub::Activity).to receive(:factory).with(instance_of(Hash), forwarder, instance_of(Hash))
 
         subject.call(json, forwarder)
       end
@@ -37,7 +37,7 @@ RSpec.describe ActivityPub::ProcessCollectionService do
         payload['signature'] = {'type' => 'RsaSignature2017'}
 
         expect_any_instance_of(ActivityPub::LinkedDataSignature).to receive(:verify_account!).and_return(actor)
-        expect(ActivityPub::Activity).to receive(:factory).with(instance_of(Hash), actor)
+        expect(ActivityPub::Activity).to receive(:factory).with(instance_of(Hash), actor, instance_of(Hash))
 
         subject.call(json, forwarder)
       end
-- 
cgit