Monsterfork v2 Kaiju Commit 2020.06.27.1 - 2020.09.05.5

9 files changed, 208 insertions, 47 deletions

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 94aee7939..337f64d53 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -4,8 +4,8 @@ class ActivityPub::Activity
   include JsonLdHelper
   include Redisable
 
-  SUPPORTED_TYPES = %w(Note Question).freeze
-  CONVERTED_TYPES = %w(Image Audio Video Article Page Event).freeze
+  SUPPORTED_TYPES = %w(Note Question Article).freeze
+  CONVERTED_TYPES = %w(Image Audio Video Page Event).freeze
 
   def initialize(json, account, **options)
     @json    = json
@@ -190,7 +190,7 @@ class ActivityPub::Activity
   end
 
   def first_local_follower
-    @account.followers.local.first
+    @account.followers.local.random.first
   end
 
   def follow_request_from_object
@@ -204,9 +204,9 @@ class ActivityPub::Activity
   def fetch_remote_original_status
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first)
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: signed_fetch_account)
     elsif @object['url'].present?
-      ::FetchRemoteStatusService.new.call(@object['url'])
+      ::FetchRemoteStatusService.new.call(@object['url'], nil, signed_fetch_account)
     end
   end
 
diff --git a/app/lib/activitypub/activity/add.rb b/app/lib/activitypub/activity/add.rb
index 688ab00b3..03b584302 100644
--- a/app/lib/activitypub/activity/add.rb
+++ b/app/lib/activitypub/activity/add.rb
@@ -9,6 +9,6 @@ class ActivityPub::Activity::Add < ActivityPub::Activity
 
     return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
 
-    StatusPin.create!(account: @account, status: status)
+    StatusPin.create(account: @account, status: status)
   end
 end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 349e8f77e..327def623 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -2,7 +2,7 @@
 
 class ActivityPub::Activity::Announce < ActivityPub::Activity
   def perform
-    return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity?
+    return reject_payload! if delete_arrived_first?(@json['id'])
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
@@ -50,7 +50,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
     elsif audience_to.include?(@account.followers_url)
       :private
     else
-      :direct
+      :limited
     end
   end
 
@@ -58,18 +58,6 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
     status.account_id == @account.id || status.distributable?
   end
 
-  def related_to_local_activity?
-    followed_by_local_accounts? || requested_through_relay? || reblog_of_local_status?
-  end
-
-  def requested_through_relay?
-    super || Relay.find_by(inbox_url: @account.inbox_url)&.enabled?
-  end
-
-  def reblog_of_local_status?
-    status_from_uri(object_uri)&.account&.local?
-  end
-
   def lock_options
     { redis: Redis.current, key: "announce:#{@object['id']}" }
   end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 3a9f83978..9d03e5247 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
+# rubocop:disable Metrics/ClassLength
 class ActivityPub::Activity::Create < ActivityPub::Activity
+  include ImgProxyHelper
+
   def perform
     dereference_object!
 
@@ -43,7 +46,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def create_status
-    return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity?
+    return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity? || twitter_retweet?
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
@@ -51,7 +54,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
         @status = find_existing_status
 
-        if @status.nil?
+        if @status.nil? || @options[:update]
           process_status
         elsif @options[:delivered_to_account_id].present?
           postprocess_audience_and_deliver
@@ -72,17 +75,33 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) }
   end
 
+  def object_uri
+    @object['id'] || super
+  end
+
   def process_status
     @tags     = []
     @mentions = []
     @params   = {}
 
-    process_status_params
+    unless @status.nil?
+      reblog_uri.blank? ? process_status_update_params : process_reblog_update_params
+      process_tags
+      process_audience
+
+      @status = UpdateStatusService.new.call(@status, @params, @mentions, @tags)
+      resolve_thread(@status)
+      fetch_replies(@status)
+      return @status
+    end
+
+    reblog_uri.blank? ? process_status_params : process_reblog_params
     process_tags
     process_audience
 
     ApplicationRecord.transaction do
       @status = Status.create!(@params)
+      process_inline_images!
       attach_tags(@status)
     end
 
@@ -108,7 +127,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         text: text_from_content || '',
         language: detected_language,
         spoiler_text: converted_object_type? ? '' : (text_from_summary || ''),
+        title: text_from_title,
+        reblog: reblogged_status,
         created_at: @object['published'],
+        expires_at: @object['expires'],
         override_timestamps: @options[:override_timestamps],
         reply: @object['inReplyTo'].present?,
         sensitive: @object['sensitive'] || false,
@@ -121,6 +143,55 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     end
   end
 
+  def process_status_update_params
+    @params = begin
+      {
+        text: text_from_content || '',
+        language: detected_language,
+        spoiler_text: converted_object_type? ? '' : (text_from_summary || ''),
+        title: text_from_title,
+        sensitive: @object['sensitive'] || false,
+        visibility: visibility_from_audience,
+        expires_at: @object['expires'],
+        media_attachment_ids: process_attachments.take(4).map(&:id),
+      }
+    end
+  end
+
+  def process_reblog_params
+    @params = begin
+      {
+        uri: object_uri,
+        url: object_url || object_uri,
+        account: @account,
+        text: text_from_content || '',
+        language: detected_language,
+        spoiler_text: converted_object_type? ? '' : (text_from_summary || ''),
+        title: text_from_title,
+        reblog: reblogged_status,
+        created_at: @object['published'],
+        override_timestamps: @options[:override_timestamps],
+        reply: @object['inReplyTo'].present?,
+        sensitive: @object['sensitive'] || false,
+        visibility: visibility_from_audience,
+        thread: replied_to_status,
+      }
+    end
+  end
+
+  def process_reblog_update_params
+    @params = begin
+      {
+        text: text_from_content || '',
+        language: detected_language,
+        spoiler_text: converted_object_type? ? '' : (text_from_summary || ''),
+        title: text_from_title,
+        sensitive: @object['sensitive'] || false,
+        visibility: visibility_from_audience,
+      }
+    end
+  end
+
   def process_audience
     (audience_to + audience_cc).uniq.each do |audience|
       next if audience == ActivityPub::TagManager::COLLECTIONS[:public]
@@ -240,7 +311,21 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
       begin
         href             = Addressable::URI.parse(attachment['url']).normalize.to_s
-        media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
+        media_attachment = MediaAttachment.find_by(account: @account, remote_url: href)
+
+        if media_attachment.nil?
+          media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
+        else
+          updated_description = attachment['summary'].presence || media_attachment[:description].presence || attachment['name'].presence || media_attachment[:name].presence
+          updated_focus = attachment['focalPoint'].presence || media_attachment['focalPoint'].presence
+          updated_blurhash = supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : media_attachment[:blurhash]
+
+          media_attachment.update(description: updated_description, focus: updated_focus, blurhash: updated_blurhash)
+
+          media_attachments << media_attachment
+          next
+        end
+
         media_attachments << media_attachment
 
         next if unsupported_media_type?(attachment['mediaType']) || skip_download?
@@ -330,22 +415,43 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def fetch_replies(status)
+    FetchReplyWorker.perform_async(@object['root']) unless invalid_root_uri?
+
     collection = @object['replies']
     return if collection.nil?
 
-    replies = ActivityPub::FetchRepliesService.new.call(status, collection, false)
-    return unless replies.nil?
-
-    uri = value_or_id(collection)
-    ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
+    if collection.is_a?(Hash)
+      ActivityPub::FetchRepliesService.new.call(status, collection)
+    else
+      uri = value_or_id(collection)
+      ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
+    end
   end
 
   def conversation_from_uri(uri)
     return nil if uri.nil?
-    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
+
+    conversation = OStatus::TagManager.instance.local_id?(uri) ? Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) : nil
 
     begin
-      Conversation.find_or_create_by!(uri: uri)
+      conversation = Conversation.find_by(uri: uri) if conversation.blank?
+
+      if @object['inReplyTo'].blank? && replied_to_status.blank?
+        params = {
+          uri: uri,
+          root: object_uri,
+          account: @account,
+        }.freeze
+        if conversation.blank?
+          conversation = Conversation.create!(params)
+        elsif conversation.root.blank?
+          conversation.update!(params)
+        end
+      elsif conversation.blank?
+        conversation = Conversation.create!(uri: uri, account_id: nil)
+      end
+
+      conversation
     rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
       retry
     end
@@ -377,7 +483,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def replied_to_status
     return @replied_to_status if defined?(@replied_to_status)
 
-    if in_reply_to_uri.blank?
+    if in_reply_to_uri.blank? || in_reply_to_uri == object_uri
       @replied_to_status = nil
     else
       @replied_to_status   = status_from_uri(in_reply_to_uri)
@@ -390,13 +496,28 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     value_or_id(@object['inReplyTo'])
   end
 
+  def reblogged_status
+    FetchRemoteStatusService.new.call(reblog_uri) if reblog_uri.present?
+  end
+
+  def reblog_uri
+    return @reblog_uri if defined?(@reblog_uri)
+
+    @reblog_uri = @object['reblog'].presence || @object['_misskey_quote'].presence
+  end
+
+  def twitter_retweet?
+    text_from_content.present? && (text_from_content.include?('<p>🐦🔗') || text_from_content.include?('<p>RT @'))
+  end
+
   def text_from_content
-    return Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type?
+    return @status_text if defined?(@status_text)
+    return @status_text = Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type?
 
     if @object['content'].present?
-      @object['content']
+      @status_text = @object['type'] == 'Article' ? Formatter.instance.format_article(@object['content']) : @object['content']
     elsif content_language_map?
-      @object['contentMap'].values.first
+      @status_text = @object['contentMap'].values.first
     end
   end
 
@@ -408,6 +529,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     end
   end
 
+  def text_from_title
+    if @object['title'].present?
+      @object['title']
+    elsif title_language_map?
+      @object['titleMap'].values.first
+    end
+  end
+
   def text_from_name
     if @object['name'].present?
       @object['name']
@@ -444,6 +573,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty?
   end
 
+  def title_language_map?
+    @object['titleMap'].is_a?(Hash) && !@object['titleMap'].empty?
+  end
+
   def content_language_map?
     @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
   end
@@ -490,6 +623,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Account.local.where(username: local_usernames).exists?
   end
 
+  def invalid_root_uri?
+    @object['root'].blank? || [object_uri, @object['url']].include?(@object['root']) || status_from_uri(@object['root'])
+  end
+
   def tombstone_exists?
     Tombstone.exists?(uri: object_uri)
   end
@@ -524,3 +661,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
   end
 end
+# rubocop:enable Metrics/ClassLength
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index dc9ff580c..1420c6aff 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -51,15 +51,12 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
   def replied_to_status
     return @replied_to_status if defined?(@replied_to_status)
-    @replied_to_status = @status.thread
-  end
 
-  def reply_to_local?
-    !replied_to_status.nil? && replied_to_status.account.local?
+    @replied_to_status = @status.thread
   end
 
   def forward_for_reply
-    return unless @json['signature'].present? && reply_to_local?
+    return if @json['signature'].blank? || replied_to_status.blank?
 
     inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url]
 
diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb
index 018e2df54..d1dba5196 100644
--- a/app/lib/activitypub/activity/update.rb
+++ b/app/lib/activitypub/activity/update.rb
@@ -2,6 +2,7 @@
 
 class ActivityPub::Activity::Update < ActivityPub::Activity
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
+  SUPPORTED_OBJECT_TYPES = (ActivityPub::Activity::SUPPORTED_TYPES + ActivityPub::Activity::CONVERTED_TYPES).freeze
 
   def perform
     dereference_object!
@@ -10,6 +11,9 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
       update_account
     elsif equals_or_includes_any?(@object['type'], %w(Question))
       update_poll
+    elsif equals_or_includes_any?(@object['type'], SUPPORTED_OBJECT_TYPES)
+      @options[:update] = true
+      ActivityPub::Activity::Create.new(@json, @account, @options).perform
     end
   end
 
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index 4e406b41d..93fd2d910 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -8,6 +8,18 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
 
   CONTEXT_EXTENSION_MAP = {
     direct_message: { 'litepub': 'http://litepub.social/ns#', 'directMessage': 'litepub:directMessage' },
+    edited: { 'mp' => 'https://the.monsterpit.net/ns#', 'edited' => 'mp:edited' },
+    require_dereference: { 'mp' => 'https://the.monsterpit.net/ns#', 'requireDereference' => 'mp:requireDereference' },
+    show_replies: { 'mp' => 'https://the.monsterpit.net/ns#', 'showReplies' => 'mp:showReplies' },
+    show_unlisted: { 'mp' => 'https://the.monsterpit.net/ns#', 'showUnlisted' => 'mp:showUnlisted' },
+    private: { 'mp' => 'https://the.monsterpit.net/ns#', 'private' => 'mp:private' },
+    require_auth: { 'mp' => 'https://the.monsterpit.net/ns#', 'requireAuth' => 'mp:requireAuth' },
+    metadata: { 'mp' => 'https://the.monsterpit.net/ns#', 'metadata' => { '@id' => 'mp:metadata', '@type' => '@id' } },
+    server_metadata: { 'mp' => 'https://the.monsterpit.net/ns#', 'serverMetadata' => { '@id' => 'mp:serverMetadata', '@type' => '@id' } },
+    root: { 'mp' => 'https://the.monsterpit.net/ns#', 'root' => { '@id' => 'mp:root', '@type' => '@id' } },
+    reblog: { 'mp' => 'https://the.monsterpit.net/ns#', 'reblog' => { '@id' => 'mp:reblog', '@type' => '@id' },
+              'misskey' => 'https://misskey.io/ns#', '_misskey_quote' => { '@id' => 'misskey:_misskey_quote', '@type' => '@id' } },
+    expires: { 'mp' => 'https://the.monsterpit.net/ns#', 'expires' => 'mp:expires' },
     manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
     sensitive: { 'sensitive' => 'as:sensitive' },
     hashtag: { 'Hashtag' => 'as:Hashtag' },
@@ -15,7 +27,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
     also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
     emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
     featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
-    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
+    property_value: { 'schema' => 'http://schema.org', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
     atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
     conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
     focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
diff --git a/app/lib/activitypub/case_transform.rb b/app/lib/activitypub/case_transform.rb
index 7f716f862..7f31fabda 100644
--- a/app/lib/activitypub/case_transform.rb
+++ b/app/lib/activitypub/case_transform.rb
@@ -14,8 +14,10 @@ module ActivityPub::CaseTransform
       when String
         camel_lower_cache[value] ||= if value.start_with?('_:')
                                        '_:' + value.gsub(/\A_:/, '').underscore.camelize(:lower)
-                                     else
+                                     elsif value != '_misskey_quote'
                                        value.underscore.camelize(:lower)
+                                     else
+                                       value
                                      end
       else value
       end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 3f98dad2e..c26301f7e 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -60,8 +60,8 @@ class ActivityPub::TagManager
   # Public statuses go out to primarily the public collection
   # Unlisted and private statuses go out primarily to the followers collection
   # Others go out only to the people they mention
-  def to(status)
-    case status.visibility
+  def to(status, target_domain: nil)
+    case status.visibility_for_domain(target_domain)
     when 'public'
       [COLLECTIONS[:public]]
     when 'unlisted', 'private'
@@ -92,19 +92,39 @@ class ActivityPub::TagManager
   # Unlisted statuses go to the public as well
   # Both of those and private statuses also go to the people mentioned in them
   # Direct ones don't have a secondary audience
-  def cc(status)
+  def cc(status, target_domain: nil)
     cc = []
 
     cc << uri_for(status.reblog.account) if status.reblog?
 
-    case status.visibility
+    visibility = status.visibility_for_domain(target_domain)
+
+    case visibility
     when 'public'
       cc << account_followers_url(status.account)
     when 'unlisted'
       cc << COLLECTIONS[:public]
+    when 'limited'
+      if status.account.silenced?
+        # Only notify followers if the account is locally silenced
+        account_ids = status.silent_mentions.pluck(:account_id)
+        cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
+          result << uri_for(account)
+          result << account_followers_url(account) if account.group?
+        end)
+        cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
+          result << uri_for(request.account)
+          result << account_followers_url(request.account) if request.account.group?
+        end)
+      else
+        cc.concat(status.silent_mentions.each_with_object([]) do |mention, result|
+          result << uri_for(mention.account)
+          result << account_followers_url(mention.account) if mention.account.group?
+        end)
+      end
     end
 
-    unless status.direct_visibility? || status.limited_visibility?
+    unless %w(direct limited).include?(visibility)
       if status.account.silenced?
         # Only notify followers if the account is locally silenced
         account_ids = status.active_mentions.pluck(:account_id)