Merge commit '4aea3f88a6d30f102a79c2da7fcfac96465ba1a8' into merging-upstream

7 files changed, 103 insertions, 40 deletions

diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index 68ca58d62..a95931afe 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -17,6 +17,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account)
     actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id) if actor.nil?
 
+    return if actor.suspended?
+
     ActivityPub::Activity.factory(activity, actor).perform
   end
 
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index b54e447ad..811209537 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -12,12 +12,21 @@ class ActivityPub::ProcessAccountService < BaseService
     @uri         = @json['id']
     @username    = username
     @domain      = domain
-    @account     = Account.find_by(uri: @uri)
     @collections = {}
 
-    create_account  if @account.nil?
-    upgrade_account if @account.ostatus?
-    update_account
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        @account        = Account.find_by(uri: @uri)
+        @old_public_key = @account&.public_key
+        @old_protocol   = @account&.protocol
+
+        create_account if @account.nil?
+        update_account
+      end
+    end
+
+    after_protocol_change! if protocol_changed?
+    after_key_change! if key_changed?
 
     @account
   rescue Oj::ParseError
@@ -35,33 +44,46 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.suspended   = true if auto_suspend?
     @account.silenced    = true if auto_silence?
     @account.private_key = nil
-    @account.save!
   end
 
   def update_account
     @account.last_webfingered_at = Time.now.utc
     @account.protocol            = :activitypub
-    @account.inbox_url           = @json['inbox'] || ''
-    @account.outbox_url          = @json['outbox'] || ''
-    @account.shared_inbox_url    = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
-    @account.followers_url       = @json['followers'] || ''
-    @account.url                 = url || @uri
-    @account.display_name        = @json['name'] || ''
-    @account.note                = @json['summary'] || ''
-    @account.avatar_remote_url   = image_url('icon')  unless skip_download?
-    @account.header_remote_url   = image_url('image') unless skip_download?
-    @account.public_key          = public_key || ''
-    @account.locked              = @json['manuallyApprovesFollowers'] || false
-    @account.statuses_count      = outbox_total_items    if outbox_total_items.present?
-    @account.following_count     = following_total_items if following_total_items.present?
-    @account.followers_count     = followers_total_items if followers_total_items.present?
+
+    set_immediate_attributes!
+    set_fetchable_attributes!
+
     @account.save_with_optional_media!
   end
 
-  def upgrade_account
+  def set_immediate_attributes!
+    @account.inbox_url        = @json['inbox'] || ''
+    @account.outbox_url       = @json['outbox'] || ''
+    @account.shared_inbox_url = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
+    @account.followers_url    = @json['followers'] || ''
+    @account.url              = url || @uri
+    @account.display_name     = @json['name'] || ''
+    @account.note             = @json['summary'] || ''
+    @account.locked           = @json['manuallyApprovesFollowers'] || false
+  end
+
+  def set_fetchable_attributes!
+    @account.avatar_remote_url = image_url('icon')  unless skip_download?
+    @account.header_remote_url = image_url('image') unless skip_download?
+    @account.public_key        = public_key || ''
+    @account.statuses_count    = outbox_total_items    if outbox_total_items.present?
+    @account.following_count   = following_total_items if following_total_items.present?
+    @account.followers_count   = followers_total_items if followers_total_items.present?
+  end
+
+  def after_protocol_change!
     ActivityPub::PostUpgradeWorker.perform_async(@account.domain)
   end
 
+  def after_key_change!
+    RefollowWorker.perform_async(@account.id)
+  end
+
   def image_url(key)
     value = first_of_value(@json[key])
 
@@ -120,15 +142,27 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def auto_suspend?
-    domain_block && domain_block.suspend?
+    domain_block&.suspend?
   end
 
   def auto_silence?
-    domain_block && domain_block.silence?
+    domain_block&.silence?
   end
 
   def domain_block
     return @domain_block if defined?(@domain_block)
     @domain_block = DomainBlock.find_by(domain: @domain)
   end
+
+  def key_changed?
+    !@old_public_key.nil? && @old_public_key != @account.public_key
+  end
+
+  def protocol_changed?
+    !@old_protocol.nil? && @old_protocol != @account.protocol
+  end
+
+  def lock_options
+    { redis: Redis.current, key: "process_account:#{@uri}" }
+  end
 end
diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb
index bc04c50ba..0c6736a3f 100644
--- a/app/services/activitypub/process_collection_service.rb
+++ b/app/services/activitypub/process_collection_service.rb
@@ -7,9 +7,9 @@ class ActivityPub::ProcessCollectionService < BaseService
     @account = account
     @json    = Oj.load(body, mode: :strict)
 
-    return if @account.suspended? || !supported_context?
-
+    return unless supported_context?
     return if different_actor? && verify_account!.nil?
+    return if @account.suspended?
 
     case @json['type']
     when 'Collection', 'CollectionPage'
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 1473bc841..eefdc0dbf 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -26,6 +26,7 @@ class BlockDomainService < BaseService
   def clear_media!
     clear_account_images
     clear_account_attachments
+    clear_emojos
   end
 
   def suspend_accounts!
@@ -51,6 +52,10 @@ class BlockDomainService < BaseService
     end
   end
 
+  def clear_emojos
+    emojis_from_blocked_domains.destroy_all
+  end
+
   def blocked_domain
     domain_block.domain
   end
@@ -62,4 +67,8 @@ class BlockDomainService < BaseService
   def media_from_blocked_domain
     MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
   end
+
+  def emojis_from_blocked_domains
+    CustomEmoji.where(domain: blocked_domain)
+  end
 end
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index c38e9e7df..4acbfae7a 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -1,7 +1,15 @@
 # frozen_string_literal: true
 
 class FetchLinkCardService < BaseService
-  URL_PATTERN = %r{https?://\S+}
+  URL_PATTERN = %r{
+    (                                                                                                 #   $1 URL
+      (https?:\/\/)?                                                                                  #   $2 Protocol (optional)
+      (#{Twitter::Regex[:valid_domain]})                                                              #   $3 Domain(s)
+      (?::(#{Twitter::Regex[:valid_port_number]}))?                                                   #   $4 Port number (optional)
+      (/#{Twitter::Regex[:valid_url_path]}*)?                                                         #   $5 URL Path and anchor
+      (\?#{Twitter::Regex[:valid_url_query_chars]}*#{Twitter::Regex[:valid_url_query_ending_chars]})? #   $6 Query String
+    )
+  }iox
 
   def call(status)
     @status = status
@@ -14,11 +22,11 @@ class FetchLinkCardService < BaseService
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @card = PreviewCard.find_by(url: @url)
-        process_url if @card.nil?
+        process_url if @card.nil? || @card.updated_at <= 2.weeks.ago
       end
     end
 
-    attach_card unless @card.nil?
+    attach_card if @card&.persisted?
   rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError
     nil
   end
@@ -26,8 +34,8 @@ class FetchLinkCardService < BaseService
   private
 
   def process_url
-    @card = PreviewCard.new(url: @url)
-    res   = Request.new(:head, @url).perform
+    @card ||= PreviewCard.new(url: @url)
+    res     = Request.new(:head, @url).perform
 
     return if res.code != 200 || res.mime_type != 'text/html'
 
@@ -40,7 +48,7 @@ class FetchLinkCardService < BaseService
 
   def parse_urls
     if @status.local?
-      urls = @status.text.match(URL_PATTERN).to_a.map { |uri| Addressable::URI.parse(uri).normalize }
+      urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[0]).normalize }
     else
       html  = Nokogiri::HTML(@status.text)
       links = html.css('a')
@@ -106,12 +114,25 @@ class FetchLinkCardService < BaseService
     guess = detector.detect(html, response.charset)
     page  = Nokogiri::HTML(html, nil, guess&.fetch(:encoding))
 
-    @card.type             = :link
-    @card.title            = meta_property(page, 'og:title') || page.at_xpath('//title')&.content || ''
-    @card.description      = meta_property(page, 'og:description') || meta_property(page, 'description') || ''
-    @card.image_remote_url = meta_property(page, 'og:image') if meta_property(page, 'og:image')
+    if meta_property(page, 'twitter:player')
+      @card.type   = :video
+      @card.width  = meta_property(page, 'twitter:player:width') || 0
+      @card.height = meta_property(page, 'twitter:player:height') || 0
+      @card.html   = content_tag(:iframe, nil, src: meta_property(page, 'twitter:player'),
+                                               width: @card.width,
+                                               height: @card.height,
+                                               allowtransparency: 'true',
+                                               scrolling: 'no',
+                                               frameborder: '0')
+    else
+      @card.type             = :link
+      @card.image_remote_url = meta_property(page, 'og:image') if meta_property(page, 'og:image')
+    end
+
+    @card.title            = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
+    @card.description      = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
 
-    return if @card.title.blank?
+    return if @card.title.blank? && @card.html.blank?
 
     @card.save_with_optional_image!
   end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index e5b0fe438..d1b8f42c7 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -28,7 +28,7 @@ class PostStatusService < BaseService
                                         sensitive: options[:sensitive],
                                         spoiler_text: options[:spoiler_text] || '',
                                         visibility: options[:visibility] || account.user&.setting_default_privacy,
-                                        language: detect_language_for(text, account),
+                                        language: LanguageDetector.instance.detect(text, account),
                                         application: options[:application])
 
       attach_media(status, media)
@@ -73,10 +73,6 @@ class PostStatusService < BaseService
     media.update(status_id: status.id)
   end
 
-  def detect_language_for(text, account)
-    LanguageDetector.new(text, account).to_iso_s
-  end
-
   def process_mentions_service
     @process_mentions_service ||= ProcessMentionsService.new
   end
diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb
index 7031c98f5..57c80fc82 100644
--- a/app/services/resolve_remote_account_service.rb
+++ b/app/services/resolve_remote_account_service.rb
@@ -80,6 +80,7 @@ class ResolveRemoteAccountService < BaseService
   def activitypub_ready?
     !@webfinger.link('self').nil? &&
       ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) &&
+      !actor_json.nil? &&
       actor_json['inbox'].present?
   end