Add support for editing for published statuses (#16697)

* Add support for editing for published statuses

* Fix references to stripped-out code

* Various fixes and improvements

* Further fixes and improvements

* Fix updates being potentially sent to unauthorized recipients

* Various fixes and improvements

* Fix wrong words in test

* Fix notifying accounts that were tagged but were not in the audience

* Fix mistake

1 files changed, 84 insertions, 165 deletions

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 8a0dc9d33..a861c34bc 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -69,9 +69,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_status
-    @tags     = []
-    @mentions = []
-    @params   = {}
+    @tags                 = []
+    @mentions             = []
+    @silenced_account_ids = []
+    @params               = {}
 
     process_status_params
     process_tags
@@ -84,10 +85,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
-    distribute(@status)
+    distribute
     forward_for_reply
   end
 
+  def distribute
+    # Spread out crawling randomly to avoid DDoSing the link
+    LinkCrawlWorker.perform_in(rand(1..59).seconds, @status.id)
+
+    # Distribute into home and list feeds and notify mentioned accounts
+    ::DistributionWorker.perform_async(@status.id, silenced_account_ids: @silenced_account_ids) if @options[:override_timestamps] || @status.within_realtime_window?
+  end
+
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
@@ -95,19 +104,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_status_params
+    @status_parser = ActivityPub::Parser::StatusParser.new(@json, followers_collection: @account.followers_url)
+
     @params = begin
       {
-        uri: object_uri,
-        url: object_url || object_uri,
+        uri: @status_parser.uri,
+        url: @status_parser.url || @status_parser.uri,
         account: @account,
-        text: text_from_content || '',
-        language: detected_language,
-        spoiler_text: converted_object_type? ? '' : (text_from_summary || ''),
-        created_at: @object['published'],
+        text: converted_object_type? ? converted_text : (@status_parser.text || ''),
+        language: @status_parser.language || detected_language,
+        spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''),
+        created_at: @status_parser.created_at,
+        edited_at: @status_parser.edited_at,
         override_timestamps: @options[:override_timestamps],
-        reply: @object['inReplyTo'].present?,
-        sensitive: @account.sensitized? || @object['sensitive'] || false,
-        visibility: visibility_from_audience,
+        reply: @status_parser.reply,
+        sensitive: @account.sensitized? || @status_parser.sensitive || false,
+        visibility: @status_parser.visibility,
         thread: replied_to_status,
         conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
@@ -117,42 +129,40 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_audience
-    (audience_to + audience_cc).uniq.each do |audience|
-      next if ActivityPub::TagManager.instance.public_collection?(audience)
+    # Unlike with tags, there is no point in resolving accounts we don't already
+    # know here, because silent mentions would only be used for local access control anyway
+    accounts_in_audience = (audience_to + audience_cc).uniq.filter_map do |audience|
+      account_from_uri(audience) unless ActivityPub::TagManager.instance.public_collection?(audience)
+    end
 
-      # Unlike with tags, there is no point in resolving accounts we don't already
-      # know here, because silent mentions would only be used for local access
-      # control anyway
-      account = account_from_uri(audience)
+    # If the payload was delivered to a specific inbox, the inbox owner must have
+    # access to it, unless they already have access to it anyway
+    if @options[:delivered_to_account_id]
+      accounts_in_audience << delivered_to_account
+      accounts_in_audience.uniq!
+    end
 
-      next if account.nil? || @mentions.any? { |mention| mention.account_id == account.id }
+    accounts_in_audience.each do |account|
+      # This runs after tags are processed, and those translate into non-silent
+      # mentions, which take precedence
+      next if @mentions.any? { |mention| mention.account_id == account.id }
 
       @mentions << Mention.new(account: account, silent: true)
 
       # If there is at least one silent mention, then the status can be considered
       # as a limited-audience status, and not strictly a direct message, but only
       # if we considered a direct message in the first place
-      next unless @params[:visibility] == :direct
-
-      @params[:visibility] = :limited
+      @params[:visibility] = :limited if @params[:visibility] == :direct
     end
 
-    # If the payload was delivered to a specific inbox, the inbox owner must have
-    # access to it, unless they already have access to it anyway
-    return if @options[:delivered_to_account_id].nil? || @mentions.any? { |mention| mention.account_id == @options[:delivered_to_account_id] }
-
-    @mentions << Mention.new(account_id: @options[:delivered_to_account_id], silent: true)
-
-    return unless @params[:visibility] == :direct
-
-    @params[:visibility] = :limited
+    # Accounts that are tagged but are not in the audience are not
+    # supposed to be notified explicitly
+    @silenced_account_ids = @mentions.map(&:account_id) - accounts_in_audience.map(&:id)
   end
 
   def postprocess_audience_and_deliver
     return if @status.mentions.find_by(account_id: @options[:delivered_to_account_id])
 
-    delivered_to_account = Account.find(@options[:delivered_to_account_id])
-
     @status.mentions.create(account: delivered_to_account, silent: true)
     @status.update(visibility: :limited) if @status.direct_visibility?
 
@@ -161,6 +171,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     FeedInsertWorker.perform_async(@status.id, delivered_to_account.id, :home)
   end
 
+  def delivered_to_account
+    @delivered_to_account ||= Account.find(@options[:delivered_to_account_id])
+  end
+
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
@@ -215,21 +229,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def process_emoji(tag)
     return if skip_download?
-    return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
 
-    shortcode = tag['name'].delete(':')
-    image_url = tag['icon']['url']
-    uri       = tag['id']
-    updated   = tag['updated']
-    emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
+    custom_emoji_parser = ActivityPub::Parser::CustomEmojiParser.new(tag)
+
+    return if custom_emoji_parser.shortcode.blank? || custom_emoji_parser.image_remote_url.blank?
 
-    return unless emoji.nil? || image_url != emoji.image_remote_url || (updated && updated >= emoji.updated_at)
+    emoji = CustomEmoji.find_by(shortcode: custom_emoji_parser.shortcode, domain: @account.domain)
 
-    emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
-    emoji.image_remote_url = image_url
-    emoji.save
-  rescue Seahorse::Client::NetworkingError => e
-    Rails.logger.warn "Error storing emoji: #{e}"
+    return unless emoji.nil? || custom_emoji_parser.image_remote_url != emoji.image_remote_url || (custom_emoji_parser.updated_at && custom_emoji_parser.updated_at >= emoji.updated_at)
+
+    begin
+      emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: custom_emoji_parser.shortcode, uri: custom_emoji_parser.uri)
+      emoji.image_remote_url = custom_emoji_parser.image_remote_url
+      emoji.save
+    rescue Seahorse::Client::NetworkingError => e
+      Rails.logger.warn "Error storing emoji: #{e}"
+    end
   end
 
   def process_attachments
@@ -238,14 +253,23 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     media_attachments = []
 
     as_array(@object['attachment']).each do |attachment|
-      next if attachment['url'].blank? || media_attachments.size >= 4
+      media_attachment_parser = ActivityPub::Parser::MediaAttachmentParser.new(attachment)
+
+      next if media_attachment_parser.remote_url.blank? || media_attachments.size >= 4
 
       begin
-        href             = Addressable::URI.parse(attachment['url']).normalize.to_s
-        media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
+        media_attachment = MediaAttachment.create(
+          account: @account,
+          remote_url: media_attachment_parser.remote_url,
+          thumbnail_remote_url: media_attachment_parser.thumbnail_remote_url,
+          description: media_attachment_parser.description,
+          focus: media_attachment_parser.focus,
+          blurhash: media_attachment_parser.blurhash
+        )
+
         media_attachments << media_attachment
 
-        next if unsupported_media_type?(attachment['mediaType']) || skip_download?
+        next if unsupported_media_type?(media_attachment_parser.file_content_type) || skip_download?
 
         media_attachment.download_file!
         media_attachment.download_thumbnail!
@@ -263,42 +287,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     media_attachments
   end
 
-  def icon_url_from_attachment(attachment)
-    url = attachment['icon'].is_a?(Hash) ? attachment['icon']['url'] : attachment['icon']
-    Addressable::URI.parse(url).normalize.to_s if url.present?
-  rescue Addressable::URI::InvalidURIError
-    nil
-  end
-
   def process_poll
-    return unless @object['type'] == 'Question' && (@object['anyOf'].is_a?(Array) || @object['oneOf'].is_a?(Array))
-
-    expires_at = begin
-      if @object['closed'].is_a?(String)
-        @object['closed']
-      elsif !@object['closed'].nil? && !@object['closed'].is_a?(FalseClass)
-        Time.now.utc
-      else
-        @object['endTime']
-      end
-    end
-
-    if @object['anyOf'].is_a?(Array)
-      multiple = true
-      items    = @object['anyOf']
-    else
-      multiple = false
-      items    = @object['oneOf']
-    end
+    poll_parser = ActivityPub::Parser::PollParser.new(@object)
 
-    voters_count = @object['votersCount']
+    return unless poll_parser.valid?
 
     @account.polls.new(
-      multiple: multiple,
-      expires_at: expires_at,
-      options: items.map { |item| item['name'].presence || item['content'] }.compact,
-      cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
-      voters_count: voters_count
+      multiple: poll_parser.multiple,
+      expires_at: poll_parser.expires_at,
+      options: poll_parser.options,
+      cached_tallies: poll_parser.cached_tallies,
+      voters_count: poll_parser.voters_count
     )
   end
 
@@ -351,23 +350,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     end
   end
 
-  def visibility_from_audience
-    if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
-      :public
-    elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
-      :unlisted
-    elsif audience_to.include?(@account.followers_url)
-      :private
-    else
-      :direct
-    end
-  end
-
-  def audience_includes?(account)
-    uri = ActivityPub::TagManager.instance.uri_for(account)
-    audience_to.include?(uri) || audience_cc.include?(uri)
-  end
-
   def replied_to_status
     return @replied_to_status if defined?(@replied_to_status)
 
@@ -384,81 +366,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     value_or_id(@object['inReplyTo'])
   end
 
-  def text_from_content
-    return Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type?
-
-    if @object['content'].present?
-      @object['content']
-    elsif content_language_map?
-      @object['contentMap'].values.first
-    end
-  end
-
-  def text_from_summary
-    if @object['summary'].present?
-      @object['summary']
-    elsif summary_language_map?
-      @object['summaryMap'].values.first
-    end
-  end
-
-  def text_from_name
-    if @object['name'].present?
-      @object['name']
-    elsif name_language_map?
-      @object['nameMap'].values.first
-    end
+  def converted_text
+    Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n"))
   end
 
   def detected_language
-    if content_language_map?
-      @object['contentMap'].keys.first
-    elsif name_language_map?
-      @object['nameMap'].keys.first
-    elsif summary_language_map?
-      @object['summaryMap'].keys.first
-    elsif supported_object_type?
-      LanguageDetector.instance.detect(text_from_content, @account)
-    end
-  end
-
-  def object_url
-    return if @object['url'].blank?
-
-    url_candidate = url_to_href(@object['url'], 'text/html')
-
-    if invalid_origin?(url_candidate)
-      nil
-    else
-      url_candidate
-    end
-  end
-
-  def summary_language_map?
-    @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty?
-  end
-
-  def content_language_map?
-    @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
-  end
-
-  def name_language_map?
-    @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty?
+    LanguageDetector.instance.detect(@status_parser.text, @account) if supported_object_type?
   end
 
   def unsupported_media_type?(mime_type)
     mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type)
   end
 
-  def supported_blurhash?(blurhash)
-    components = blurhash.blank? || !blurhash_valid_chars?(blurhash) ? nil : Blurhash.components(blurhash)
-    components.present? && components.none? { |comp| comp > 5 }
-  end
-
-  def blurhash_valid_chars?(blurhash)
-    /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
-  end
-
   def skip_download?
     return @skip_download if defined?(@skip_download)