Merge pull request #932 from ThibG/glitch-soc/merge-upstream

Merge upstream changes

16 files changed, 206 insertions, 17 deletions

diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 99c16157f..6f56a67ba 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -18,6 +18,7 @@ class StatusesController < ApplicationController
   before_action :redirect_to_original, only: [:show]
   before_action :set_referrer_policy_header, only: [:show]
   before_action :set_cache_headers
+  before_action :set_replies, only: [:replies]
 
   content_security_policy only: :embed do |p|
     p.frame_ancestors(false)
@@ -65,8 +66,37 @@ class StatusesController < ApplicationController
     render 'stream_entries/embed', layout: 'embedded'
   end
 
+  def replies
+    skip_session!
+
+    render json: replies_collection_presenter,
+           serializer: ActivityPub::CollectionSerializer,
+           adapter: ActivityPub::Adapter,
+           content_type: 'application/activity+json',
+           skip_activities: true
+  end
+
   private
 
+  def replies_collection_presenter
+    page = ActivityPub::CollectionPresenter.new(
+      id: replies_account_status_url(@account, @status, page_params),
+      type: :unordered,
+      part_of: replies_account_status_url(@account, @status),
+      next: next_page,
+      items: @replies.map { |status| status.local ? status : status.id }
+    )
+    if page_requested?
+      page
+    else
+      ActivityPub::CollectionPresenter.new(
+        id: replies_account_status_url(@account, @status),
+        type: :unordered,
+        first: page
+      )
+    end
+  end
+
   def create_descendant_thread(starting_depth, statuses)
     depth = starting_depth + statuses.size
     if depth < DESCENDANTS_DEPTH_LIMIT
@@ -176,4 +206,27 @@ class StatusesController < ApplicationController
     return if @status.public_visibility? || @status.unlisted_visibility?
     response.headers['Referrer-Policy'] = 'origin'
   end
+
+  def page_requested?
+    params[:page] == 'true'
+  end
+
+  def set_replies
+    @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses
+    @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted])
+    @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id])
+  end
+
+  def next_page
+    last_reply = @replies.last
+    return if last_reply.nil?
+    same_account = last_reply.account_id == @account.id
+    return unless same_account || @replies.size == DESCENDANTS_LIMIT
+    same_account = false unless @replies.size == DESCENDANTS_LIMIT
+    replies_account_status_url(@account, @status, page: true, min_id: last_reply.id, other_accounts: !same_account)
+  end
+
+  def page_params
+    { page: true, other_accounts: params[:other_accounts], min_id: params[:min_id] }.compact
+  end
 end
diff --git a/app/javascript/mastodon/actions/conversations.js b/app/javascript/mastodon/actions/conversations.js
index 3c2ea9680..c6e062ef7 100644
--- a/app/javascript/mastodon/actions/conversations.js
+++ b/app/javascript/mastodon/actions/conversations.js
@@ -41,13 +41,15 @@ export const expandConversations = ({ maxId } = {}) => (dispatch, getState) => {
     params.since_id = getState().getIn(['conversations', 'items', 0, 'last_status']);
   }
 
+  const isLoadingRecent = !!params.since_id;
+
   api(getState).get('/api/v1/conversations', { params })
     .then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
 
       dispatch(importFetchedAccounts(response.data.reduce((aggr, item) => aggr.concat(item.accounts), [])));
       dispatch(importFetchedStatuses(response.data.map(item => item.last_status).filter(x => !!x)));
-      dispatch(expandConversationsSuccess(response.data, next ? next.uri : null));
+      dispatch(expandConversationsSuccess(response.data, next ? next.uri : null, isLoadingRecent));
     })
     .catch(err => dispatch(expandConversationsFail(err)));
 };
@@ -56,10 +58,11 @@ export const expandConversationsRequest = () => ({
   type: CONVERSATIONS_FETCH_REQUEST,
 });
 
-export const expandConversationsSuccess = (conversations, next) => ({
+export const expandConversationsSuccess = (conversations, next, isLoadingRecent) => ({
   type: CONVERSATIONS_FETCH_SUCCESS,
   conversations,
   next,
+  isLoadingRecent,
 });
 
 export const expandConversationsFail = error => ({
diff --git a/app/javascript/mastodon/reducers/conversations.js b/app/javascript/mastodon/reducers/conversations.js
index 955a07754..9564bffcd 100644
--- a/app/javascript/mastodon/reducers/conversations.js
+++ b/app/javascript/mastodon/reducers/conversations.js
@@ -35,7 +35,7 @@ const updateConversation = (state, item) => state.update('items', list => {
   }
 });
 
-const expandNormalizedConversations = (state, conversations, next) => {
+const expandNormalizedConversations = (state, conversations, next, isLoadingRecent) => {
   let items = ImmutableList(conversations.map(conversationToMap));
 
   return state.withMutations(mutable => {
@@ -66,7 +66,7 @@ const expandNormalizedConversations = (state, conversations, next) => {
       });
     }
 
-    if (!next) {
+    if (!next && !isLoadingRecent) {
       mutable.set('hasMore', false);
     }
 
@@ -81,7 +81,7 @@ export default function conversations(state = initialState, action) {
   case CONVERSATIONS_FETCH_FAIL:
     return state.set('isLoading', false);
   case CONVERSATIONS_FETCH_SUCCESS:
-    return expandNormalizedConversations(state, action.conversations, action.next);
+    return expandNormalizedConversations(state, action.conversations, action.next, action.isLoadingRecent);
   case CONVERSATIONS_UPDATE:
     return updateConversation(state, action.conversation);
   case CONVERSATIONS_MOUNT:
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 11823a45b..5eef07a6e 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -2336,6 +2336,7 @@ a.account__display-name {
 
 .getting-started {
   color: $dark-text-color;
+  overflow: auto;
 
   &__footer {
     flex: 0 0 auto;
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index d7bd65c80..0980f94ba 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -40,6 +40,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     end
 
     resolve_thread(@status)
+    fetch_replies(@status)
     distribute(@status)
     forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
   end
@@ -159,7 +160,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if tag['href'].blank?
 
     account = account_from_uri(tag['href'])
-    account = ::FetchRemoteAccountService.new.call(tag['href'], id: false) if account.nil?
+    account = ::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
 
     return if account.nil?
 
@@ -213,6 +214,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
   end
 
+  def fetch_replies(status)
+    collection = @object['replies']
+    return if collection.nil?
+    replies = ActivityPub::FetchRepliesService.new.call(status, collection, false)
+    return if replies.present?
+    uri = value_or_id(collection)
+    ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
+  end
+
   def conversation_from_uri(uri)
     return nil if uri.nil?
     return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index be3a562d0..892bb9974 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -48,6 +48,12 @@ class ActivityPub::TagManager
     activity_account_status_url(target.account, target)
   end
 
+  def replies_uri_for(target, page_params = nil)
+    raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
+
+    replies_account_status_url(target.account, target, page_params)
+  end
+
   # Primary audience of a status
   # Public statuses go out to primarily the public collection
   # Unlisted and private statuses go out primarily to the followers collection
diff --git a/app/models/concerns/status_threading_concern.rb b/app/models/concerns/status_threading_concern.rb
index b9c800c2a..15eb695cd 100644
--- a/app/models/concerns/status_threading_concern.rb
+++ b/app/models/concerns/status_threading_concern.rb
@@ -11,6 +11,10 @@ module StatusThreadingConcern
     find_statuses_from_tree_path(descendant_ids(limit, max_child_id, since_child_id, depth), account, promote: true)
   end
 
+  def self_replies(limit)
+    account.statuses.where(in_reply_to_id: id, visibility: [:public, :unlisted]).reorder(id: :asc).limit(limit)
+  end
+
   private
 
   def ancestor_ids(limit)
diff --git a/app/presenters/activitypub/collection_presenter.rb b/app/presenters/activitypub/collection_presenter.rb
index ec84ab1a3..28331f0c4 100644
--- a/app/presenters/activitypub/collection_presenter.rb
+++ b/app/presenters/activitypub/collection_presenter.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class ActivityPub::CollectionPresenter < ActiveModelSerializers::Model
-  attributes :id, :type, :size, :items, :part_of, :first, :last, :next, :prev
+  attributes :id, :type, :size, :items, :page, :part_of, :first, :last, :next, :prev
 end
diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb
index b51e8c544..c001e28aa 100644
--- a/app/serializers/activitypub/activity_serializer.rb
+++ b/app/serializers/activitypub/activity_serializer.rb
@@ -3,8 +3,8 @@
 class ActivityPub::ActivitySerializer < ActiveModel::Serializer
   attributes :id, :type, :actor, :published, :to, :cc
 
-  has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, unless: :owned_announce?
-  attribute :proper_uri, key: :object, if: :owned_announce?
+  has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, if: :serialize_object?
+  attribute :proper_uri, key: :object, unless: :serialize_object?
   attribute :atom_uri, if: :announce?
 
   def id
@@ -43,7 +43,9 @@ class ActivityPub::ActivitySerializer < ActiveModel::Serializer
     object.reblog?
   end
 
-  def owned_announce?
-    announce? && object.account == object.proper.account && object.proper.private_visibility?
+  def serialize_object?
+    return true unless announce?
+    # Serialize private self-boosts of local toots
+    object.account == object.proper.account && object.proper.private_visibility? && object.local?
   end
 end
diff --git a/app/serializers/activitypub/collection_serializer.rb b/app/serializers/activitypub/collection_serializer.rb
index e8960131b..b03609957 100644
--- a/app/serializers/activitypub/collection_serializer.rb
+++ b/app/serializers/activitypub/collection_serializer.rb
@@ -7,7 +7,8 @@ class ActivityPub::CollectionSerializer < ActiveModel::Serializer
     super
   end
 
-  attributes :id, :type
+  attribute :id, if: -> { object.id.present? }
+  attribute :type
   attribute :total_items, if: -> { object.size.present? }
   attribute :next, if: -> { object.next.present? }
   attribute :prev, if: -> { object.prev.present? }
@@ -37,6 +38,6 @@ class ActivityPub::CollectionSerializer < ActiveModel::Serializer
   end
 
   def page?
-    object.part_of.present?
+    object.part_of.present? || object.page.present?
   end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index c9d23e25f..4aab993a9 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -13,6 +13,8 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
   has_many :media_attachments, key: :attachment
   has_many :virtual_tags, key: :tag
 
+  has_one :replies, serializer: ActivityPub::CollectionSerializer, if: :local?
+
   def id
     ActivityPub::TagManager.instance.uri_for(object)
   end
@@ -33,6 +35,21 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
     { object.language => Formatter.instance.format(object) }
   end
 
+  def replies
+    replies = object.self_replies(5).pluck(:id, :uri)
+    last_id = replies.last&.first
+    ActivityPub::CollectionPresenter.new(
+      type: :unordered,
+      id: ActivityPub::TagManager.instance.replies_uri_for(object),
+      first: ActivityPub::CollectionPresenter.new(
+        type: :unordered,
+        part_of: ActivityPub::TagManager.instance.replies_uri_for(object),
+        items: replies.map(&:second),
+        next: last_id ? ActivityPub::TagManager.instance.replies_uri_for(object, page: true, min_id: last_id) : nil
+      )
+    )
+  end
+
   def language?
     object.language.present?
   end
diff --git a/app/services/activitypub/fetch_replies_service.rb b/app/services/activitypub/fetch_replies_service.rb
new file mode 100644
index 000000000..95c486a43
--- /dev/null
+++ b/app/services/activitypub/fetch_replies_service.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class ActivityPub::FetchRepliesService < BaseService
+  include JsonLdHelper
+
+  def call(parent_status, collection_or_uri, allow_synchronous_requests = true)
+    @account = parent_status.account
+    @allow_synchronous_requests = allow_synchronous_requests
+
+    @items = collection_items(collection_or_uri)
+    return if @items.nil?
+
+    FetchReplyWorker.push_bulk(filtered_replies)
+
+    @items
+  end
+
+  private
+
+  def collection_items(collection_or_uri)
+    collection = fetch_collection(collection_or_uri)
+    return unless collection.is_a?(Hash)
+
+    collection = fetch_collection(collection['first']) if collection['first'].present?
+    return unless collection.is_a?(Hash)
+
+    case collection['type']
+    when 'Collection', 'CollectionPage'
+      collection['items']
+    when 'OrderedCollection', 'OrderedCollectionPage'
+      collection['orderedItems']
+    end
+  end
+
+  def fetch_collection(collection_or_uri)
+    return collection_or_uri if collection_or_uri.is_a?(Hash)
+    return unless @allow_synchronous_requests
+    return if invalid_origin?(collection_or_uri)
+    collection = fetch_resource_without_id_validation(collection_or_uri)
+    raise Mastodon::UnexpectedResponseError if collection.nil?
+    collection
+  end
+
+  def filtered_replies
+    # Only fetch replies to the same server as the original status to avoid
+    # amplification attacks.
+
+    # Also limit to 5 fetched replies to limit potential for DoS.
+    @items.map { |item| value_or_id(item) }.reject { |uri| invalid_origin?(uri) }.take(5)
+  end
+
+  def invalid_origin?(url)
+    return true if unsupported_uri_scheme?(url)
+
+    needle   = Addressable::URI.parse(url).host
+    haystack = Addressable::URI.parse(@account.uri).host
+
+    !haystack.casecmp(needle).zero?
+  end
+end
diff --git a/app/workers/activitypub/fetch_replies_worker.rb b/app/workers/activitypub/fetch_replies_worker.rb
new file mode 100644
index 000000000..bf466db54
--- /dev/null
+++ b/app/workers/activitypub/fetch_replies_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class ActivityPub::FetchRepliesWorker
+  include Sidekiq::Worker
+  include ExponentialBackoff
+
+  sidekiq_options queue: 'pull', retry: 3
+
+  def perform(parent_status_id, replies_uri)
+    ActivityPub::FetchRepliesService.new.call(Status.find(parent_status_id), replies_uri)
+  end
+end
diff --git a/app/workers/concerns/exponential_backoff.rb b/app/workers/concerns/exponential_backoff.rb
new file mode 100644
index 000000000..f2b931e33
--- /dev/null
+++ b/app/workers/concerns/exponential_backoff.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ExponentialBackoff
+  extend ActiveSupport::Concern
+
+  included do
+    sidekiq_retry_in do |count|
+      15 + 10 * (count**4) + rand(10 * (count**4))
+    end
+  end
+end
diff --git a/app/workers/fetch_reply_worker.rb b/app/workers/fetch_reply_worker.rb
new file mode 100644
index 000000000..f7aa25e81
--- /dev/null
+++ b/app/workers/fetch_reply_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class FetchReplyWorker
+  include Sidekiq::Worker
+  include ExponentialBackoff
+
+  sidekiq_options queue: 'pull', retry: 3
+
+  def perform(child_url)
+    FetchRemoteStatusService.new.call(child_url)
+  end
+end
diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb
index c18a778d5..8bba9ca75 100644
--- a/app/workers/thread_resolve_worker.rb
+++ b/app/workers/thread_resolve_worker.rb
@@ -2,13 +2,10 @@
 
 class ThreadResolveWorker
   include Sidekiq::Worker
+  include ExponentialBackoff
 
   sidekiq_options queue: 'pull', retry: 3
 
-  sidekiq_retry_in do |count|
-    15 + 10 * (count**4) + rand(10 * (count**4))
-  end
-
   def perform(child_status_id, parent_url)
     child_status  = Status.find(child_status_id)
     parent_status = FetchRemoteStatusService.new.call(parent_url)