Merge pull request #157 from glitch-soc/merging-upstream

ABRACA-HRRRRRRRRRRRNGGGGGGGHHH!!!!!!!!!!!!!!!!!!!

16 files changed, 134 insertions, 58 deletions

diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index 68ca58d62..a95931afe 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -17,6 +17,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account)
     actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id) if actor.nil?
 
+    return if actor.suspended?
+
     ActivityPub::Activity.factory(activity, actor).perform
   end
 
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index b54e447ad..811209537 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -12,12 +12,21 @@ class ActivityPub::ProcessAccountService < BaseService
     @uri         = @json['id']
     @username    = username
     @domain      = domain
-    @account     = Account.find_by(uri: @uri)
     @collections = {}
 
-    create_account  if @account.nil?
-    upgrade_account if @account.ostatus?
-    update_account
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        @account        = Account.find_by(uri: @uri)
+        @old_public_key = @account&.public_key
+        @old_protocol   = @account&.protocol
+
+        create_account if @account.nil?
+        update_account
+      end
+    end
+
+    after_protocol_change! if protocol_changed?
+    after_key_change! if key_changed?
 
     @account
   rescue Oj::ParseError
@@ -35,33 +44,46 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.suspended   = true if auto_suspend?
     @account.silenced    = true if auto_silence?
     @account.private_key = nil
-    @account.save!
   end
 
   def update_account
     @account.last_webfingered_at = Time.now.utc
     @account.protocol            = :activitypub
-    @account.inbox_url           = @json['inbox'] || ''
-    @account.outbox_url          = @json['outbox'] || ''
-    @account.shared_inbox_url    = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
-    @account.followers_url       = @json['followers'] || ''
-    @account.url                 = url || @uri
-    @account.display_name        = @json['name'] || ''
-    @account.note                = @json['summary'] || ''
-    @account.avatar_remote_url   = image_url('icon')  unless skip_download?
-    @account.header_remote_url   = image_url('image') unless skip_download?
-    @account.public_key          = public_key || ''
-    @account.locked              = @json['manuallyApprovesFollowers'] || false
-    @account.statuses_count      = outbox_total_items    if outbox_total_items.present?
-    @account.following_count     = following_total_items if following_total_items.present?
-    @account.followers_count     = followers_total_items if followers_total_items.present?
+
+    set_immediate_attributes!
+    set_fetchable_attributes!
+
     @account.save_with_optional_media!
   end
 
-  def upgrade_account
+  def set_immediate_attributes!
+    @account.inbox_url        = @json['inbox'] || ''
+    @account.outbox_url       = @json['outbox'] || ''
+    @account.shared_inbox_url = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
+    @account.followers_url    = @json['followers'] || ''
+    @account.url              = url || @uri
+    @account.display_name     = @json['name'] || ''
+    @account.note             = @json['summary'] || ''
+    @account.locked           = @json['manuallyApprovesFollowers'] || false
+  end
+
+  def set_fetchable_attributes!
+    @account.avatar_remote_url = image_url('icon')  unless skip_download?
+    @account.header_remote_url = image_url('image') unless skip_download?
+    @account.public_key        = public_key || ''
+    @account.statuses_count    = outbox_total_items    if outbox_total_items.present?
+    @account.following_count   = following_total_items if following_total_items.present?
+    @account.followers_count   = followers_total_items if followers_total_items.present?
+  end
+
+  def after_protocol_change!
     ActivityPub::PostUpgradeWorker.perform_async(@account.domain)
   end
 
+  def after_key_change!
+    RefollowWorker.perform_async(@account.id)
+  end
+
   def image_url(key)
     value = first_of_value(@json[key])
 
@@ -120,15 +142,27 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def auto_suspend?
-    domain_block && domain_block.suspend?
+    domain_block&.suspend?
   end
 
   def auto_silence?
-    domain_block && domain_block.silence?
+    domain_block&.silence?
   end
 
   def domain_block
     return @domain_block if defined?(@domain_block)
     @domain_block = DomainBlock.find_by(domain: @domain)
   end
+
+  def key_changed?
+    !@old_public_key.nil? && @old_public_key != @account.public_key
+  end
+
+  def protocol_changed?
+    !@old_protocol.nil? && @old_protocol != @account.protocol
+  end
+
+  def lock_options
+    { redis: Redis.current, key: "process_account:#{@uri}" }
+  end
 end
diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb
index bc04c50ba..59cb65c65 100644
--- a/app/services/activitypub/process_collection_service.rb
+++ b/app/services/activitypub/process_collection_service.rb
@@ -7,9 +7,9 @@ class ActivityPub::ProcessCollectionService < BaseService
     @account = account
     @json    = Oj.load(body, mode: :strict)
 
-    return if @account.suspended? || !supported_context?
-
+    return unless supported_context?
     return if different_actor? && verify_account!.nil?
+    return if @account.suspended? || @account.local?
 
     case @json['type']
     when 'Collection', 'CollectionPage'
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 86eaa5735..2fd623922 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -18,7 +18,7 @@ class BatchedRemoveStatusService < BaseService
     @stream_entry_batches  = []
     @salmon_batches        = []
     @activity_json_batches = []
-    @json_payloads         = statuses.map { |s| [s.id, Oj.dump(event: :delete, payload: s.id)] }.to_h
+    @json_payloads         = statuses.map { |s| [s.id, Oj.dump(event: :delete, payload: s.id.to_s)] }.to_h
     @activity_json         = {}
     @activity_xml          = {}
 
@@ -84,6 +84,8 @@ class BatchedRemoveStatusService < BaseService
   end
 
   def unpush_from_public_timelines(status)
+    return unless status.public_visibility?
+
     payload = @json_payloads[status.id]
 
     redis.pipelined do
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 1473bc841..eefdc0dbf 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -26,6 +26,7 @@ class BlockDomainService < BaseService
   def clear_media!
     clear_account_images
     clear_account_attachments
+    clear_emojos
   end
 
   def suspend_accounts!
@@ -51,6 +52,10 @@ class BlockDomainService < BaseService
     end
   end
 
+  def clear_emojos
+    emojis_from_blocked_domains.destroy_all
+  end
+
   def blocked_domain
     domain_block.domain
   end
@@ -62,4 +67,8 @@ class BlockDomainService < BaseService
   def media_from_blocked_domain
     MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
   end
+
+  def emojis_from_blocked_domains
+    CustomEmoji.where(domain: blocked_domain)
+  end
 end
diff --git a/app/services/concerns/author_extractor.rb b/app/services/concerns/author_extractor.rb
index 867d6dc25..c2366188a 100644
--- a/app/services/concerns/author_extractor.rb
+++ b/app/services/concerns/author_extractor.rb
@@ -5,12 +5,12 @@ module AuthorExtractor
     return nil if xml.nil?
 
     # Try <email> for acct
-    acct = xml.at_xpath('./xmlns:author/xmlns:email', xmlns: TagManager::XMLNS)&.content
+    acct = xml.at_xpath('./xmlns:author/xmlns:email', xmlns: OStatus::TagManager::XMLNS)&.content
 
     # Try <name> + <uri>
     if acct.blank?
-      username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: TagManager::XMLNS)&.content
-      uri      = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: TagManager::XMLNS)&.content
+      username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: OStatus::TagManager::XMLNS)&.content
+      uri      = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: OStatus::TagManager::XMLNS)&.content
 
       return nil if username.blank? || uri.blank?
 
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index c38e9e7df..4acbfae7a 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -1,7 +1,15 @@
 # frozen_string_literal: true
 
 class FetchLinkCardService < BaseService
-  URL_PATTERN = %r{https?://\S+}
+  URL_PATTERN = %r{
+    (                                                                                                 #   $1 URL
+      (https?:\/\/)?                                                                                  #   $2 Protocol (optional)
+      (#{Twitter::Regex[:valid_domain]})                                                              #   $3 Domain(s)
+      (?::(#{Twitter::Regex[:valid_port_number]}))?                                                   #   $4 Port number (optional)
+      (/#{Twitter::Regex[:valid_url_path]}*)?                                                         #   $5 URL Path and anchor
+      (\?#{Twitter::Regex[:valid_url_query_chars]}*#{Twitter::Regex[:valid_url_query_ending_chars]})? #   $6 Query String
+    )
+  }iox
 
   def call(status)
     @status = status
@@ -14,11 +22,11 @@ class FetchLinkCardService < BaseService
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @card = PreviewCard.find_by(url: @url)
-        process_url if @card.nil?
+        process_url if @card.nil? || @card.updated_at <= 2.weeks.ago
       end
     end
 
-    attach_card unless @card.nil?
+    attach_card if @card&.persisted?
   rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError
     nil
   end
@@ -26,8 +34,8 @@ class FetchLinkCardService < BaseService
   private
 
   def process_url
-    @card = PreviewCard.new(url: @url)
-    res   = Request.new(:head, @url).perform
+    @card ||= PreviewCard.new(url: @url)
+    res     = Request.new(:head, @url).perform
 
     return if res.code != 200 || res.mime_type != 'text/html'
 
@@ -40,7 +48,7 @@ class FetchLinkCardService < BaseService
 
   def parse_urls
     if @status.local?
-      urls = @status.text.match(URL_PATTERN).to_a.map { |uri| Addressable::URI.parse(uri).normalize }
+      urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[0]).normalize }
     else
       html  = Nokogiri::HTML(@status.text)
       links = html.css('a')
@@ -106,12 +114,25 @@ class FetchLinkCardService < BaseService
     guess = detector.detect(html, response.charset)
     page  = Nokogiri::HTML(html, nil, guess&.fetch(:encoding))
 
-    @card.type             = :link
-    @card.title            = meta_property(page, 'og:title') || page.at_xpath('//title')&.content || ''
-    @card.description      = meta_property(page, 'og:description') || meta_property(page, 'description') || ''
-    @card.image_remote_url = meta_property(page, 'og:image') if meta_property(page, 'og:image')
+    if meta_property(page, 'twitter:player')
+      @card.type   = :video
+      @card.width  = meta_property(page, 'twitter:player:width') || 0
+      @card.height = meta_property(page, 'twitter:player:height') || 0
+      @card.html   = content_tag(:iframe, nil, src: meta_property(page, 'twitter:player'),
+                                               width: @card.width,
+                                               height: @card.height,
+                                               allowtransparency: 'true',
+                                               scrolling: 'no',
+                                               frameborder: '0')
+    else
+      @card.type             = :link
+      @card.image_remote_url = meta_property(page, 'og:image') if meta_property(page, 'og:image')
+    end
+
+    @card.title            = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
+    @card.description      = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
 
-    return if @card.title.blank?
+    return if @card.title.blank? && @card.html.blank?
 
     @card.save_with_optional_image!
   end
diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb
index 7c618a0b0..bd98e70d1 100644
--- a/app/services/fetch_remote_account_service.rb
+++ b/app/services/fetch_remote_account_service.rb
@@ -25,7 +25,7 @@ class FetchRemoteAccountService < BaseService
     xml = Nokogiri::XML(body)
     xml.encoding = 'utf-8'
 
-    account = author_from_xml(xml.at_xpath('/xmlns:feed', xmlns: TagManager::XMLNS), false)
+    account = author_from_xml(xml.at_xpath('/xmlns:feed', xmlns: OStatus::TagManager::XMLNS), false)
 
     UpdateRemoteProfileService.new.call(xml, account) unless account.nil?
 
diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index 18af18059..1b90854c4 100644
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -27,7 +27,7 @@ class FetchRemoteStatusService < BaseService
     xml = Nokogiri::XML(body)
     xml.encoding = 'utf-8'
 
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: TagManager::XMLNS))
+    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
     domain  = Addressable::URI.parse(url).normalized_host
 
     return nil unless !account.nil? && confirmed_domain?(domain, account)
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index e5b0fe438..d1b8f42c7 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -28,7 +28,7 @@ class PostStatusService < BaseService
                                         sensitive: options[:sensitive],
                                         spoiler_text: options[:spoiler_text] || '',
                                         visibility: options[:visibility] || account.user&.setting_default_privacy,
-                                        language: detect_language_for(text, account),
+                                        language: LanguageDetector.instance.detect(text, account),
                                         application: options[:application])
 
       attach_media(status, media)
@@ -73,10 +73,6 @@ class PostStatusService < BaseService
     media.update(status_id: status.id)
   end
 
-  def detect_language_for(text, account)
-    LanguageDetector.new(text, account).to_iso_s
-  end
-
   def process_mentions_service
     @process_mentions_service ||= ProcessMentionsService.new
   end
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
index 31191a818..2a5f1e2bc 100644
--- a/app/services/process_feed_service.rb
+++ b/app/services/process_feed_service.rb
@@ -16,7 +16,7 @@ class ProcessFeedService < BaseService
   end
 
   def process_entries(xml, account)
-    xml.xpath('//xmlns:entry', xmlns: TagManager::XMLNS).reverse_each.map { |entry| process_entry(entry, account) }.compact
+    xml.xpath('//xmlns:entry', xmlns: OStatus::TagManager::XMLNS).reverse_each.map { |entry| process_entry(entry, account) }.compact
   end
 
   def process_entry(xml, account)
diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb
index d04e926e7..1fca3832b 100644
--- a/app/services/process_interaction_service.rb
+++ b/app/services/process_interaction_service.rb
@@ -13,7 +13,7 @@ class ProcessInteractionService < BaseService
     xml = Nokogiri::XML(body)
     xml.encoding = 'utf-8'
 
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: TagManager::XMLNS))
+    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
 
     return if account.nil? || account.suspended?
 
@@ -54,13 +54,13 @@ class ProcessInteractionService < BaseService
   private
 
   def mentions_account?(xml, account)
-    xml.xpath('/xmlns:entry/xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each { |mention_link| return true if [TagManager.instance.uri_for(account), TagManager.instance.url_for(account)].include?(mention_link.attribute('href').value) }
+    xml.xpath('/xmlns:entry/xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each { |mention_link| return true if [OStatus::TagManager.instance.uri_for(account), OStatus::TagManager.instance.url_for(account)].include?(mention_link.attribute('href').value) }
     false
   end
 
   def verb(xml)
-    raw = xml.at_xpath('//activity:verb', activity: TagManager::AS_XMLNS).content
-    TagManager::VERBS.key(raw)
+    raw = xml.at_xpath('//activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
+    OStatus::TagManager::VERBS.key(raw)
   rescue
     :post
   end
@@ -104,7 +104,7 @@ class ProcessInteractionService < BaseService
   end
 
   def delete_post!(xml, account)
-    status = Status.find(xml.at_xpath('//xmlns:id', xmlns: TagManager::XMLNS).content)
+    status = Status.find(xml.at_xpath('//xmlns:id', xmlns: OStatus::TagManager::XMLNS).content)
 
     return if status.nil?
 
@@ -137,12 +137,12 @@ class ProcessInteractionService < BaseService
 
   def status(xml)
     uri = activity_id(xml)
-    return nil unless TagManager.instance.local_id?(uri)
-    Status.find(TagManager.instance.unique_tag_to_local_id(uri, 'Status'))
+    return nil unless OStatus::TagManager.instance.local_id?(uri)
+    Status.find(OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status'))
   end
 
   def activity_id(xml)
-    xml.at_xpath('//activity:object', activity: TagManager::AS_XMLNS).at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content
+    xml.at_xpath('//activity:object', activity: OStatus::TagManager::AS_XMLNS).at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
   end
 
   def salmon
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index f123bf869..1c3eea369 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -39,7 +39,7 @@ class ProcessMentionsService < BaseService
 
     if mentioned_account.local?
       NotifyService.new.call(mentioned_account, mention)
-    elsif mentioned_account.ostatus? && (Rails.configuration.x.use_ostatus_privacy || !status.stream_entry.hidden?)
+    elsif mentioned_account.ostatus? && !status.stream_entry.hidden?
       NotificationWorker.perform_async(stream_entry_to_xml(status.stream_entry), status.account_id, mentioned_account.id)
     elsif mentioned_account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(build_json(mention.status), mention.status.account_id, mentioned_account.inbox_url)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 83fc77043..14f24908c 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -4,7 +4,7 @@ class RemoveStatusService < BaseService
   include StreamEntryRenderer
 
   def call(status)
-    @payload      = Oj.dump(event: :delete, payload: status.id)
+    @payload      = Oj.dump(event: :delete, payload: status.id.to_s)
     @status       = status
     @account      = status.account
     @tags         = status.tags.pluck(:name).to_a
@@ -14,6 +14,7 @@ class RemoveStatusService < BaseService
 
     remove_from_self if status.account.local?
     remove_from_followers
+    remove_from_affected
     remove_reblogs
     remove_from_hashtags
     remove_from_public
@@ -38,6 +39,12 @@ class RemoveStatusService < BaseService
     end
   end
 
+  def remove_from_affected
+    @mentions.map(&:account).select(&:local?).each do |account|
+      Redis.current.publish("timeline:#{account.id}", @payload)
+    end
+  end
+
   def remove_from_remote_affected
     # People who got mentioned in the status, or who
     # reblogged it from someone else might not follow
@@ -105,6 +112,8 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_hashtags
+    return unless @status.public_visibility?
+
     @tags.each do |hashtag|
       Redis.current.publish("timeline:hashtag:#{hashtag}", @payload)
       Redis.current.publish("timeline:hashtag:#{hashtag}:local", @payload) if @status.local?
@@ -112,6 +121,8 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_public
+    return unless @status.public_visibility?
+
     Redis.current.publish('timeline:public', @payload)
     Redis.current.publish('timeline:public:local', @payload) if @status.local?
   end
diff --git a/app/services/resolve_remote_account_service.rb b/app/services/resolve_remote_account_service.rb
index 7031c98f5..57c80fc82 100644
--- a/app/services/resolve_remote_account_service.rb
+++ b/app/services/resolve_remote_account_service.rb
@@ -80,6 +80,7 @@ class ResolveRemoteAccountService < BaseService
   def activitypub_ready?
     !@webfinger.link('self').nil? &&
       ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) &&
+      !actor_json.nil? &&
       actor_json['inbox'].present?
   end
 
diff --git a/app/services/verify_salmon_service.rb b/app/services/verify_salmon_service.rb
index cd674837d..205b35d8b 100644
--- a/app/services/verify_salmon_service.rb
+++ b/app/services/verify_salmon_service.rb
@@ -9,7 +9,7 @@ class VerifySalmonService < BaseService
     xml = Nokogiri::XML(body)
     xml.encoding = 'utf-8'
 
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: TagManager::XMLNS))
+    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
 
     if account.nil?
       false