From dd7ef0dc41584089a97444d8192bc61505108e6c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 8 Aug 2017 21:52:15 +0200
Subject: Add ActivityPub inbox (#4216)

* Add ActivityPub inbox

* Handle ActivityPub deletes

* Handle ActivityPub creates

* Handle ActivityPub announces

* Stubs for handling all activities that need to be handled

* Add ActivityPub actor resolving

* Handle conversation URI passing in ActivityPub

* Handle content language in ActivityPub

* Send accept header when fetching actor, handle JSON parse errors

* Test for ActivityPub::FetchRemoteAccountService

* Handle public key and icon/image when embedded/as array/as resolvable URI

* Implement ActivityPub::FetchRemoteStatusService

* Add stubs for more interactions

* Undo activities implemented

* Handle out of order activities

* Hook up ActivityPub to ResolveRemoteAccountService, handle
Update Account activities

* Add fragment IDs to all transient activity serializers

* Add tests and fixes

* Add stubs for missing tests

* Add more tests

* Add more tests
---
 app/lib/activitypub/activity/create.rb | 148 +++++++++++++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100644 app/lib/activitypub/activity/create.rb

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
new file mode 100644
index 000000000..4c4049bc6
--- /dev/null
+++ b/app/lib/activitypub/activity/create.rb
@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+class ActivityPub::Activity::Create < ActivityPub::Activity
+  def perform
+    return if delete_arrived_first?(object_uri) || unsupported_object_type?
+
+    status = Status.find_by(uri: object_uri)
+
+    return status unless status.nil?
+
+    ApplicationRecord.transaction do
+      status = Status.create!(status_params)
+
+      process_tags(status)
+      process_attachments(status)
+    end
+
+    resolve_thread(status)
+    distribute(status)
+
+    status
+  end
+
+  private
+
+  def status_params
+    {
+      uri: @object['id'],
+      url: @object['url'],
+      account: @account,
+      text: text_from_content || '',
+      language: language_from_content,
+      spoiler_text: @object['summary'] || '',
+      created_at: @object['published'] || Time.now.utc,
+      reply: @object['inReplyTo'].present?,
+      sensitive: @object['sensitive'] || false,
+      visibility: visibility_from_audience,
+      thread: replied_to_status,
+      conversation: conversation_from_uri(@object['_:conversation']),
+    }
+  end
+
+  def process_tags(status)
+    return unless @object['tag'].is_a?(Array)
+
+    @object['tag'].each do |tag|
+      case tag['type']
+      when 'Hashtag'
+        process_hashtag tag, status
+      when 'Mention'
+        process_mention tag, status
+      end
+    end
+  end
+
+  def process_hashtag(tag, status)
+    hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
+    hashtag = Tag.where(name: hashtag).first_or_initialize(name: hashtag)
+
+    status.tags << hashtag
+  end
+
+  def process_mention(tag, status)
+    account = account_from_uri(tag['href'])
+    account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
+    return if account.nil?
+    account.mentions.create(status: status)
+  end
+
+  def process_attachments(status)
+    return unless @object['attachment'].is_a?(Array)
+
+    @object['attachment'].each do |attachment|
+      next if unsupported_media_type?(attachment['mediaType'])
+
+      href             = Addressable::URI.parse(attachment['url']).normalize.to_s
+      media_attachment = MediaAttachment.create(status: status, account: status.account, remote_url: href)
+
+      next if skip_download?
+
+      media_attachment.file_remote_url = href
+      media_attachment.save
+    end
+  end
+
+  def resolve_thread(status)
+    return unless status.reply? && status.thread.nil?
+    ActivityPub::ThreadResolveWorker.perform_async(status.id, @object['inReplyTo'])
+  end
+
+  def conversation_from_uri(uri)
+    return nil if uri.nil?
+    return Conversation.find_by(id: TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if TagManager.instance.local_id?(uri)
+    Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
+  end
+
+  def visibility_from_audience
+    if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
+      :public
+    elsif equals_or_includes?(@object['cc'], ActivityPub::TagManager::COLLECTIONS[:public])
+      :unlisted
+    elsif equals_or_includes?(@object['to'], @account.followers_url)
+      :private
+    else
+      :direct
+    end
+  end
+
+  def audience_includes?(account)
+    uri = ActivityPub::TagManager.instance.uri_for(account)
+    equals_or_includes?(@object['to'], uri) || equals_or_includes?(@object['cc'], uri)
+  end
+
+  def replied_to_status
+    return if @object['inReplyTo'].blank?
+    @replied_to_status ||= status_from_uri(@object['inReplyTo'])
+  end
+
+  def text_from_content
+    if @object['content'].present?
+      @object['content']
+    elsif language_map?
+      @object['contentMap'].values.first
+    end
+  end
+
+  def language_from_content
+    return nil unless language_map?
+    @object['contentMap'].keys.first
+  end
+
+  def language_map?
+    @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
+  end
+
+  def unsupported_object_type?
+    @object.is_a?(String) || !%w(Article Note).include?(@object['type'])
+  end
+
+  def unsupported_media_type?(mime_type)
+    mime_type.present? && !(MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES).include?(mime_type)
+  end
+
+  def skip_download?
+    return @skip_download if defined?(@skip_download)
+    @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media?
+  end
+end
-- 
cgit 


From ad892dbc0c377d82b2cae349f189a41708a1e6af Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 17 Aug 2017 21:35:00 +0200
Subject: Add _:atomUri property for deduplicating OStatus/ActivityPub legacy
 records (#4593)

---
 app/lib/activitypub/activity/create.rb         |  8 +++++++-
 app/serializers/activitypub/note_serializer.rb | 10 ++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 4c4049bc6..de9c47d5c 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -4,7 +4,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def perform
     return if delete_arrived_first?(object_uri) || unsupported_object_type?
 
-    status = Status.find_by(uri: object_uri)
+    status = find_existing_status
 
     return status unless status.nil?
 
@@ -23,6 +23,12 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   private
 
+  def find_existing_status
+    status   = Status.find_by(uri: object_uri)
+    status ||= Status.find_by(uri: @object['_:atomUri']) if @object['_:atomUri'].present?
+    status
+  end
+
   def status_params
     {
       uri: @object['id'],
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 4c13f8e59..bc8eb8a35 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -8,6 +8,8 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
   has_many :media_attachments, key: :attachment
   has_many :virtual_tags, key: :tag
 
+  attribute :atom_uri, key: '_:atomUri', if: :local?
+
   def id
     ActivityPub::TagManager.instance.uri_for(object)
   end
@@ -52,6 +54,14 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
     object.mentions + object.tags
   end
 
+  def atom_uri
+    ::TagManager.instance.uri_for(object)
+  end
+
+  def local?
+    object.account.local?
+  end
+
   class MediaAttachmentSerializer < ActiveModel::Serializer
     include RoutingHelper
 
-- 
cgit 


From 10489b4e4ab75c72f3fb49761cb7e08151752a6e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 18 Aug 2017 02:29:12 +0200
Subject: If url attribute not present in Note, fallback to id attribute
 (#4629)

---
 app/lib/activitypub/activity/create.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index de9c47d5c..77d66fba3 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -32,7 +32,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def status_params
     {
       uri: @object['id'],
-      url: @object['url'],
+      url: @object['url'] || @object['id'],
       account: @account,
       text: text_from_content || '',
       language: language_from_content,
-- 
cgit 


From 412ea873060da4dc73236fdd63a2931d27dbfa40 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 19 Aug 2017 18:44:48 +0200
Subject: Improve ActivityPub/OStatus compatibility (#4632)

*Note: OStatus URIs are invalid for ActivityPub. But we have them for
as long as we want to keep old OStatus-sourced content and as long as
we remain OStatus-compatible.*

- In Announce handling, if object URI is not a URL, fallback to object URL
- Do not use specialized ThreadResolveWorker, rely on generalized handling
- When serializing notes, if parent's URI is not a URL, use parent's URL
---
 app/lib/activitypub/activity/announce.rb               | 14 ++++++++++++--
 app/lib/activitypub/activity/create.rb                 |  2 +-
 app/serializers/activitypub/note_serializer.rb         |  8 +++++++-
 app/workers/activitypub/thread_resolve_worker.rb       | 17 -----------------
 spec/workers/activitypub/thread_resolve_worker_spec.rb | 16 ----------------
 5 files changed, 20 insertions(+), 37 deletions(-)
 delete mode 100644 app/workers/activitypub/thread_resolve_worker.rb
 delete mode 100644 spec/workers/activitypub/thread_resolve_worker_spec.rb

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index decf8f960..09fec28a0 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -2,8 +2,8 @@
 
 class ActivityPub::Activity::Announce < ActivityPub::Activity
   def perform
-    original_status = status_from_uri(object_uri)
-    original_status = ActivityPub::FetchRemoteStatusService.new.call(object_uri) if original_status.nil?
+    original_status   = status_from_uri(object_uri)
+    original_status ||= fetch_remote_original_status
 
     return if original_status.nil? || delete_arrived_first?(@json['id'])
 
@@ -11,4 +11,14 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
     distribute(status)
     status
   end
+
+  private
+
+  def fetch_remote_original_status
+    if object_uri.start_with?('http')
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri)
+    elsif @object['url'].present?
+      ::FetchRemoteStatusService.new.call(@object['url'])
+    end
+  end
 end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 77d66fba3..154125759 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -91,7 +91,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def resolve_thread(status)
     return unless status.reply? && status.thread.nil?
-    ActivityPub::ThreadResolveWorker.perform_async(status.id, @object['inReplyTo'])
+    ThreadResolveWorker.perform_async(status.id, @object['inReplyTo'])
   end
 
   def conversation_from_uri(uri)
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index bc8eb8a35..4061b9ce4 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -27,7 +27,13 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
   end
 
   def in_reply_to
-    ActivityPub::TagManager.instance.uri_for(object.thread) if object.reply?
+    return unless object.reply?
+
+    if object.thread.uri.nil? || object.thread.uri.start_with?('http')
+      ActivityPub::TagManager.instance.uri_for(object.thread)
+    else
+      object.thread.url
+    end
   end
 
   def published
diff --git a/app/workers/activitypub/thread_resolve_worker.rb b/app/workers/activitypub/thread_resolve_worker.rb
deleted file mode 100644
index 4ef762d06..000000000
--- a/app/workers/activitypub/thread_resolve_worker.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-class ActivityPub::ThreadResolveWorker
-  include Sidekiq::Worker
-
-  sidekiq_options queue: 'pull', retry: false
-
-  def perform(child_status_id, parent_uri)
-    child_status  = Status.find(child_status_id)
-    parent_status = ActivityPub::FetchRemoteStatusService.new.call(parent_uri)
-
-    return if parent_status.nil?
-
-    child_status.thread = parent_status
-    child_status.save!
-  end
-end
diff --git a/spec/workers/activitypub/thread_resolve_worker_spec.rb b/spec/workers/activitypub/thread_resolve_worker_spec.rb
deleted file mode 100644
index b954cb62c..000000000
--- a/spec/workers/activitypub/thread_resolve_worker_spec.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-require 'rails_helper'
-
-describe ActivityPub::ThreadResolveWorker do
-  subject { described_class.new }
-
-  let(:status) { Fabricate(:status) }
-  let(:parent) { Fabricate(:status) }
-
-  describe '#perform' do
-    it 'gets parent from ActivityPub::FetchRemoteStatusService and glues them together' do
-      allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(double(:service, call: parent))
-      subject.perform(status.id, 'http://example.com/123')
-      expect(status.reload.in_reply_to_id).to eq parent.id
-    end
-  end
-end
-- 
cgit 


From d63de55ef84eea883b72a121d680b8841af8e2c0 Mon Sep 17 00:00:00 2001
From: unarist <m.unarist@gmail.com>
Date: Wed, 23 Aug 2017 01:30:15 +0900
Subject: Fix bugs which OStatus accounts may detected as ActivityPub ready
 (#4662)

* Fallback to OStatus in FetchAtomService

* Skip activity+json link if that activity is Person without inbox
* If unsupported activity was detected and all other URLs failed, retry with ActivityPub-less Accept header

* Allow mention to OStatus account in ActivityPub

* Don't update profile with inbox-less Person object
---
 app/lib/activitypub/activity/create.rb             |  2 +-
 .../activitypub/process_account_service.rb         |  2 +
 app/services/fetch_atom_service.rb                 | 60 +++++++++++++---------
 .../fetch_remote_account_service_spec.rb           | 27 ++++++++++
 4 files changed, 67 insertions(+), 24 deletions(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 154125759..5c59c4b24 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -68,7 +68,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def process_mention(tag, status)
     account = account_from_uri(tag['href'])
-    account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
+    account = FetchRemoteAccountService.new.call(tag['href']) if account.nil?
     return if account.nil?
     account.mentions.create(status: status)
   end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 2f2dfd330..99f9dbdc2 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -6,6 +6,8 @@ class ActivityPub::ProcessAccountService < BaseService
   # Should be called with confirmed valid JSON
   # and WebFinger-resolved username and domain
   def call(username, domain, json)
+    return unless json['inbox'].present?
+
     @json     = json
     @uri      = @json['id']
     @username = username
diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb
index c6a4dc2e9..3cf39e006 100644
--- a/app/services/fetch_atom_service.rb
+++ b/app/services/fetch_atom_service.rb
@@ -1,13 +1,17 @@
 # frozen_string_literal: true
 
 class FetchAtomService < BaseService
+  include JsonLdHelper
+
   def call(url)
     return if url.blank?
 
-    @url = url
+    result = process(url)
 
-    perform_request
-    process_response
+    # retry without ActivityPub
+    result ||= process(url) if @unsupported_activity
+
+    result
   rescue OpenSSL::SSL::SSLError => e
     Rails.logger.debug "SSL error: #{e}"
     nil
@@ -18,9 +22,18 @@ class FetchAtomService < BaseService
 
   private
 
+  def process(url, terminal = false)
+    @url = url
+    perform_request
+    process_response(terminal)
+  end
+
   def perform_request
+    accept = 'text/html'
+    accept = 'application/activity+json, application/ld+json, application/atom+xml, ' + accept unless @unsupported_activity
+
     @response = Request.new(:get, @url)
-                       .add_headers('Accept' => 'application/activity+json, application/ld+json, application/atom+xml, text/html')
+                       .add_headers('Accept' => accept)
                        .perform
   end
 
@@ -30,7 +43,12 @@ class FetchAtomService < BaseService
     if @response.mime_type == 'application/atom+xml'
       [@url, @response.to_s, :ostatus]
     elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@response.mime_type)
-      [@url, @response.to_s, :activitypub]
+      if supported_activity?(@response.to_s)
+        [@url, @response.to_s, :activitypub]
+      else
+        @unsupported_activity = true
+        nil
+      end
     elsif @response['Link'] && !terminal
       process_headers
     elsif @response.mime_type == 'text/html' && !terminal
@@ -44,15 +62,10 @@ class FetchAtomService < BaseService
     json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
     atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }
 
-    if !json_link.nil?
-      @url = json_link['href']
-      perform_request
-      process_response(true)
-    elsif !atom_link.nil?
-      @url = atom_link['href']
-      perform_request
-      process_response(true)
-    end
+    result ||= process(json_link.href, terminal: true) unless json_link.nil? || @unsupported_activity
+    result ||= process(atom_link.href, terminal: true) unless atom_link.nil?
+
+    result
   end
 
   def process_headers
@@ -61,14 +74,15 @@ class FetchAtomService < BaseService
     json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])
     atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml))
 
-    if !json_link.nil?
-      @url = json_link.href
-      perform_request
-      process_response(true)
-    elsif !atom_link.nil?
-      @url = atom_link.href
-      perform_request
-      process_response(true)
-    end
+    result ||= process(json_link.href, terminal: true) unless json_link.nil? || @unsupported_activity
+    result ||= process(atom_link.href, terminal: true) unless atom_link.nil?
+
+    result
+  end
+
+  def supported_activity?(body)
+    json = body_to_json(body)
+    return false if json.nil? || !supported_context?(json)
+    json['type'] == 'Person' ? json['inbox'].present? : true
   end
 end
diff --git a/spec/services/activitypub/fetch_remote_account_service_spec.rb b/spec/services/activitypub/fetch_remote_account_service_spec.rb
index 786d7f7f2..391d051c1 100644
--- a/spec/services/activitypub/fetch_remote_account_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_account_service_spec.rb
@@ -11,6 +11,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService do
       preferredUsername: 'alice',
       name: 'Alice',
       summary: 'Foo bar',
+      inbox: 'http://example.com/alice/inbox',
     }
   end
 
@@ -35,6 +36,32 @@ RSpec.describe ActivityPub::FetchRemoteAccountService do
       end
     end
 
+    context 'when the account does not have a inbox' do
+      let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/alice' }] } }
+
+      before do
+        actor[:inbox] = nil
+        
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
+      end
+
+      it 'fetches resource' do
+        account
+        expect(a_request(:get, 'https://example.com/alice')).to have_been_made.once
+      end
+
+      it 'looks up webfinger' do
+        account
+        expect(a_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com')).to have_been_made.once
+      end
+
+      it 'returns nil' do
+        expect(account).to be_nil
+      end
+
+    end
+
     context 'when URI and WebFinger share the same host' do
       let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/alice' }] } }
 
-- 
cgit 


From 0d5d11eeff048a5022a6eef68d299856f5bb9860 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 26 Aug 2017 19:55:10 +0200
Subject: Add _:inReplyToAtomUri to ActivityPub (#4702)

---
 app/helpers/jsonld_helper.rb                       |  2 +-
 app/lib/activitypub/activity/create.rb             | 17 ++++++++++++++---
 app/serializers/activitypub/activity_serializer.rb |  6 +++++-
 app/serializers/activitypub/note_serializer.rb     |  7 +++++++
 4 files changed, 27 insertions(+), 5 deletions(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 09446c8be..d82a07332 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -10,7 +10,7 @@ module JsonLdHelper
   end
 
   def value_or_id(value)
-    value.is_a?(String) ? value : value['id']
+    value.is_a?(String) || value.nil? ? value : value['id']
   end
 
   def supported_context?(json)
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 5c59c4b24..114aed84f 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -91,7 +91,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def resolve_thread(status)
     return unless status.reply? && status.thread.nil?
-    ThreadResolveWorker.perform_async(status.id, @object['inReplyTo'])
+    ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
   end
 
   def conversation_from_uri(uri)
@@ -118,8 +118,19 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def replied_to_status
-    return if @object['inReplyTo'].blank?
-    @replied_to_status ||= status_from_uri(@object['inReplyTo'])
+    return @replied_to_status if defined?(@replied_to_status)
+
+    if in_reply_to_uri.blank?
+      @replied_to_status = nil
+    else
+      @replied_to_status   = status_from_uri(in_reply_to_uri)
+      @replied_to_status ||= status_from_uri(@object['_:inReplyToAtomUri']) if @object['_:inReplyToAtomUri'].present?
+      @replied_to_status
+    end
+  end
+
+  def in_reply_to_uri
+    value_or_id(@object['inReplyTo'])
   end
 
   def text_from_content
diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb
index d20ee9920..349495e84 100644
--- a/app/serializers/activitypub/activity_serializer.rb
+++ b/app/serializers/activitypub/activity_serializer.rb
@@ -10,7 +10,7 @@ class ActivityPub::ActivitySerializer < ActiveModel::Serializer
   end
 
   def type
-    object.reblog? ? 'Announce' : 'Create'
+    announce? ? 'Announce' : 'Create'
   end
 
   def actor
@@ -24,4 +24,8 @@ class ActivityPub::ActivitySerializer < ActiveModel::Serializer
   def cc
     ActivityPub::TagManager.instance.cc(object)
   end
+
+  def announce?
+    object.reblog?
+  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 4061b9ce4..15031dfdc 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -9,6 +9,7 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
   has_many :virtual_tags, key: :tag
 
   attribute :atom_uri, key: '_:atomUri', if: :local?
+  attribute :in_reply_to_atom_uri, key: '_:inReplyToAtomUri'
 
   def id
     ActivityPub::TagManager.instance.uri_for(object)
@@ -64,6 +65,12 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
     ::TagManager.instance.uri_for(object)
   end
 
+  def in_reply_to_atom_uri
+    return unless object.reply?
+
+    ::TagManager.instance.uri_for(object.thread)
+  end
+
   def local?
     object.account.local?
   end
-- 
cgit 


From 7b8f26284072120701289f90bc6602ce918e4304 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 30 Aug 2017 15:37:02 +0200
Subject: Forward ActivityPub creates that reply to local statuses (#4709)

* Forward ActivityPub creates that reply to local statuses

* Fix test

* Fix wrong signers
---
 app/lib/activitypub/activity/create.rb             | 10 ++++++
 app/lib/activitypub/activity/delete.rb             |  2 ++
 app/services/post_status_service.rb                |  1 +
 .../activitypub/reply_distribution_worker.rb       | 42 ++++++++++++++++++++++
 spec/lib/activitypub/activity/delete_spec.rb       |  1 +
 5 files changed, 56 insertions(+)
 create mode 100644 app/workers/activitypub/reply_distribution_worker.rb

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 114aed84f..2eea1827a 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -17,6 +17,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(status)
     distribute(status)
+    forward_for_reply if status.public_visibility? || status.unlisted_visibility?
 
     status
   end
@@ -162,4 +163,13 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return @skip_download if defined?(@skip_download)
     @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media?
   end
+
+  def reply_to_local?
+    !replied_to_status.nil? && replied_to_status.account.local?
+  end
+
+  def forward_for_reply
+    return unless @json['signature'].present? && reply_to_local?
+    ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id)
+  end
 end
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 789ed58f1..afa9a8079 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -16,6 +16,8 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   private
 
   def forward_for_reblogs(status)
+    return if @json['signature'].blank?
+
     ActivityPub::RawDistributionWorker.push_bulk(status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)) do |account_id|
       [payload, account_id]
     end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 5ff93f21e..568f5a9e7 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -40,6 +40,7 @@ class PostStatusService < BaseService
     DistributionWorker.perform_async(status.id)
     Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)
     ActivityPub::DistributionWorker.perform_async(status.id)
+    ActivityPub::ReplyDistributionWorker.perform_async(status.id) if status.reply? && status.thread.account.local?
 
     if options[:idempotency].present?
       redis.setex("idempotency:status:#{account.id}:#{options[:idempotency]}", 3_600, status.id)
diff --git a/app/workers/activitypub/reply_distribution_worker.rb b/app/workers/activitypub/reply_distribution_worker.rb
new file mode 100644
index 000000000..f9127340f
--- /dev/null
+++ b/app/workers/activitypub/reply_distribution_worker.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class ActivityPub::ReplyDistributionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'push'
+
+  def perform(status_id)
+    @status  = Status.find(status_id)
+    @account = @status.thread.account
+
+    return if skip_distribution?
+
+    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
+      [signed_payload, @status.account_id, inbox_url]
+    end
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+
+  private
+
+  def skip_distribution?
+    @status.private_visibility? || @status.direct_visibility?
+  end
+
+  def inboxes
+    @inboxes ||= @account.followers.inboxes
+  end
+
+  def signed_payload
+    @signed_payload ||= Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@status.account))
+  end
+
+  def payload
+    @payload ||= ActiveModelSerializers::SerializableResource.new(
+      @status,
+      serializer: ActivityPub::ActivitySerializer,
+      adapter: ActivityPub::Adapter
+    ).as_json
+  end
+end
diff --git a/spec/lib/activitypub/activity/delete_spec.rb b/spec/lib/activitypub/activity/delete_spec.rb
index 6601f7262..65e743abb 100644
--- a/spec/lib/activitypub/activity/delete_spec.rb
+++ b/spec/lib/activitypub/activity/delete_spec.rb
@@ -11,6 +11,7 @@ RSpec.describe ActivityPub::Activity::Delete do
       type: 'Delete',
       actor: ActivityPub::TagManager.instance.uri_for(sender),
       object: ActivityPub::TagManager.instance.uri_for(status),
+      signature: 'foo',
     }.with_indifferent_access
   end
 
-- 
cgit 


From 8fd8f81ae7ab7c02c4cb75c3c65ff3ed8c193f38 Mon Sep 17 00:00:00 2001
From: unarist <m.unarist@gmail.com>
Date: Sat, 2 Sep 2017 04:00:43 +0900
Subject: Deduplicate with local status on Create activity (#4763)

---
 app/lib/activitypub/activity/create.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 2eea1827a..a6f1e60d9 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -25,7 +25,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   private
 
   def find_existing_status
-    status   = Status.find_by(uri: object_uri)
+    status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['_:atomUri']) if @object['_:atomUri'].present?
     status
   end
-- 
cgit 


From 1b5806b74475213ae45b483d08a29daa40988f84 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 2 Sep 2017 14:01:23 +0200
Subject: Define missing JSON-LD properties (#4767)

Using _: property names is discouraged, as in the future,
canonicalization may throw an error when encountering that instead
of discarding it silently like it does now.

We are defining some ActivityStreams properties which we expect
to land in ActivityStreams eventually, to ensure that future versions
of Mastodon will remain compatible with this even once that happens.
Those would be `locked`, `sensitive` and `Hashtag`

We are defining a custom context inline for some properties which we
do not expect to land in any other context. `atomUri`, `inReplyToAtomUri`
and `conversation` are part of the custom defined OStatus context.
---
 app/lib/activitypub/activity/create.rb               |  6 +++---
 app/lib/activitypub/activity/delete.rb               |  2 +-
 app/lib/activitypub/adapter.rb                       | 20 +++++++++++++++++++-
 app/serializers/activitypub/actor_serializer.rb      |  4 +---
 app/serializers/activitypub/delete_serializer.rb     |  3 +--
 app/serializers/activitypub/note_serializer.rb       | 17 +++++++++++++----
 .../activitypub/fetch_remote_status_service.rb       |  4 ++--
 app/services/activitypub/process_account_service.rb  |  2 +-
 8 files changed, 41 insertions(+), 17 deletions(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index a6f1e60d9..081e80570 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -26,7 +26,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def find_existing_status
     status   = status_from_uri(object_uri)
-    status ||= Status.find_by(uri: @object['_:atomUri']) if @object['_:atomUri'].present?
+    status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
     status
   end
 
@@ -43,7 +43,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,
       thread: replied_to_status,
-      conversation: conversation_from_uri(@object['_:conversation']),
+      conversation: conversation_from_uri(@object['conversation']),
     }
   end
 
@@ -125,7 +125,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       @replied_to_status = nil
     else
       @replied_to_status   = status_from_uri(in_reply_to_uri)
-      @replied_to_status ||= status_from_uri(@object['_:inReplyToAtomUri']) if @object['_:inReplyToAtomUri'].present?
+      @replied_to_status ||= status_from_uri(@object['inReplyToAtomUri']) if @object['inReplyToAtomUri'].present?
       @replied_to_status
     end
   end
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 9d804c86d..4c6afb090 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -17,7 +17,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
   def delete_note
     status   = Status.find_by(uri: object_uri, account: @account)
-    status ||= Status.find_by(uri: @object['_:atomUri'], account: @account) if @object.is_a?(Hash) && @object['_:atomUri'].present?
+    status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
 
     delete_later!(object_uri)
 
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index 92210579e..fe4dddd38 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -1,6 +1,24 @@
 # frozen_string_literal: true
 
 class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
+  CONTEXT = {
+    '@context': [
+      'https://www.w3.org/ns/activitystreams',
+      'https://w3id.org/security/v1',
+
+      {
+        'locked'    => 'as:locked',
+        'sensitive' => 'as:sensitive',
+        'Hashtag'   => 'as:Hashtag',
+
+        'ostatus'          => 'http://ostatus.org#',
+        'atomUri'          => 'ostatus:atomUri',
+        'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri',
+        'conversation'     => 'ostatus:conversation',
+      },
+    ],
+  }.freeze
+
   def self.default_key_transform
     :camel_lower
   end
@@ -11,7 +29,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
 
   def serializable_hash(options = nil)
     options = serialization_options(options)
-    serialized_hash = { '@context': [ActivityPub::TagManager::CONTEXT, 'https://w3id.org/security/v1'] }.merge(ActiveModelSerializers::Adapter::Attributes.new(serializer, instance_options).serializable_hash(options))
+    serialized_hash = CONTEXT.merge(ActiveModelSerializers::Adapter::Attributes.new(serializer, instance_options).serializable_hash(options))
     self.class.transform_key_casing!(serialized_hash, instance_options)
   end
 end
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index a72ecee24..f004dc326 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -6,12 +6,10 @@ class ActivityPub::ActorSerializer < ActiveModel::Serializer
   attributes :id, :type, :following, :followers,
              :inbox, :outbox, :shared_inbox,
              :preferred_username, :name, :summary,
-             :url
+             :url, :locked
 
   has_one :public_key, serializer: ActivityPub::PublicKeySerializer
 
-  attribute :locked, key: '_:locked'
-
   class ImageSerializer < ActiveModel::Serializer
     include RoutingHelper
 
diff --git a/app/serializers/activitypub/delete_serializer.rb b/app/serializers/activitypub/delete_serializer.rb
index a041c577b..87a43b95d 100644
--- a/app/serializers/activitypub/delete_serializer.rb
+++ b/app/serializers/activitypub/delete_serializer.rb
@@ -2,8 +2,7 @@
 
 class ActivityPub::DeleteSerializer < ActiveModel::Serializer
   class TombstoneSerializer < ActiveModel::Serializer
-    attributes :id, :type
-    attribute :atom_uri, key: '_:atomUri'
+    attributes :id, :type, :atom_uri
 
     def id
       ActivityPub::TagManager.instance.uri_for(object)
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 15031dfdc..d42f54263 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -3,14 +3,13 @@
 class ActivityPub::NoteSerializer < ActiveModel::Serializer
   attributes :id, :type, :summary, :content,
              :in_reply_to, :published, :url,
-             :attributed_to, :to, :cc, :sensitive
+             :attributed_to, :to, :cc, :sensitive,
+             :atom_uri, :in_reply_to_atom_uri,
+             :conversation
 
   has_many :media_attachments, key: :attachment
   has_many :virtual_tags, key: :tag
 
-  attribute :atom_uri, key: '_:atomUri', if: :local?
-  attribute :in_reply_to_atom_uri, key: '_:inReplyToAtomUri'
-
   def id
     ActivityPub::TagManager.instance.uri_for(object)
   end
@@ -62,6 +61,8 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
   end
 
   def atom_uri
+    return unless object.local?
+
     ::TagManager.instance.uri_for(object)
   end
 
@@ -71,6 +72,14 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
     ::TagManager.instance.uri_for(object.thread)
   end
 
+  def conversation
+    if object.conversation.uri?
+      object.conversation.uri
+    else
+      TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
+    end
+  end
+
   def local?
     object.account.local?
   end
diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index c114515cd..68ca58d62 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -25,8 +25,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   def activity_json
     if %w(Note Article).include? @json['type']
       {
-        'type' => 'Create',
-        'actor' => first_of_value(@json['attributedTo']),
+        'type'   => 'Create',
+        'actor'  => first_of_value(@json['attributedTo']),
         'object' => @json,
       }
     else
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 99f9dbdc2..44798d051 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -50,7 +50,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.avatar_remote_url   = image_url('icon')
     @account.header_remote_url   = image_url('image')
     @account.public_key          = public_key || ''
-    @account.locked              = @json['_:locked'] || false
+    @account.locked              = @json['locked'] || false
     @account.save!
   end
 
-- 
cgit 


From 9b50a9dd835c3a08effc86a6ef3e29e3a16e3d27 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 4 Sep 2017 18:26:33 +0200
Subject: Fix some ActivityPub JSON bugs (#4796)

- Fix assumption that `url` is always a string. Handle it if it's an
  array of strings, array of objects, object, or string, both for
  accounts and for objects
- `sharedInbox` is actually supposed to be under `endpoints`, handle
  both cases and adjust the serializer
---
 app/lib/activitypub/activity/create.rb              | 12 +++++++++++-
 app/serializers/activitypub/actor_serializer.rb     | 18 +++++++++++++++---
 app/services/activitypub/process_account_service.rb | 18 ++++++++++++++----
 3 files changed, 40 insertions(+), 8 deletions(-)

(limited to 'app/lib/activitypub/activity/create.rb')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 081e80570..9a34484f5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -33,7 +33,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def status_params
     {
       uri: @object['id'],
-      url: @object['url'] || @object['id'],
+      url: object_url || @object['id'],
       account: @account,
       text: text_from_content || '',
       language: language_from_content,
@@ -147,6 +147,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     @object['contentMap'].keys.first
   end
 
+  def object_url
+    return if @object['url'].blank?
+
+    value = first_of_value(@object['url'])
+
+    return value if value.is_a?(String)
+
+    value['href']
+  end
+
   def language_map?
     @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
   end
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 25521eca9..a11178f5b 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -4,7 +4,7 @@ class ActivityPub::ActorSerializer < ActiveModel::Serializer
   include RoutingHelper
 
   attributes :id, :type, :following, :followers,
-             :inbox, :outbox, :shared_inbox,
+             :inbox, :outbox,
              :preferred_username, :name, :summary,
              :url, :manually_approves_followers
 
@@ -24,6 +24,18 @@ class ActivityPub::ActorSerializer < ActiveModel::Serializer
     end
   end
 
+  class EndpointsSerializer < ActiveModel::Serializer
+    include RoutingHelper
+
+    attributes :shared_inbox
+
+    def shared_inbox
+      inbox_url
+    end
+  end
+
+  has_one :endpoints, serializer: EndpointsSerializer
+
   has_one :icon,  serializer: ImageSerializer, if: :avatar_exists?
   has_one :image, serializer: ImageSerializer, if: :header_exists?
 
@@ -51,8 +63,8 @@ class ActivityPub::ActorSerializer < ActiveModel::Serializer
     account_outbox_url(object)
   end
 
-  def shared_inbox
-    inbox_url
+  def endpoints
+    object
   end
 
   def preferred_username
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index a26b39cb5..29eb1c2e1 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -6,7 +6,7 @@ class ActivityPub::ProcessAccountService < BaseService
   # Should be called with confirmed valid JSON
   # and WebFinger-resolved username and domain
   def call(username, domain, json)
-    return unless json['inbox'].present?
+    return if json['inbox'].blank?
 
     @json     = json
     @uri      = @json['id']
@@ -42,9 +42,9 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.protocol            = :activitypub
     @account.inbox_url           = @json['inbox'] || ''
     @account.outbox_url          = @json['outbox'] || ''
-    @account.shared_inbox_url    = @json['sharedInbox'] || ''
+    @account.shared_inbox_url    = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
     @account.followers_url       = @json['followers'] || ''
-    @account.url                 = @json['url'] || @uri
+    @account.url                 = url || @uri
     @account.display_name        = @json['name'] || ''
     @account.note                = @json['summary'] || ''
     @account.avatar_remote_url   = image_url('icon')
@@ -62,7 +62,7 @@ class ActivityPub::ProcessAccountService < BaseService
     value = first_of_value(@json[key])
 
     return if value.nil?
-    return @json[key]['url'] if @json[key].is_a?(Hash)
+    return value['url'] if value.is_a?(Hash)
 
     image = fetch_resource(value)
     image['url'] if image
@@ -78,6 +78,16 @@ class ActivityPub::ProcessAccountService < BaseService
     key['publicKeyPem'] if key
   end
 
+  def url
+    return if @json['url'].blank?
+
+    value = first_of_value(@json['url'])
+
+    return value if value.is_a?(String)
+
+    value['href']
+  end
+
   def auto_suspend?
     domain_block && domain_block.suspend?
   end
-- 
cgit