Merge pull request #3 from tootsuite/master

Updating to current

17 files changed, 128 insertions, 71 deletions

diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb
new file mode 100644
index 000000000..8c6197f2c
--- /dev/null
+++ b/app/services/after_block_service.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class AfterBlockService < BaseService
+  def call(account, target_account)
+    clear_timelines(account, target_account)
+    clear_notifications(account, target_account)
+  end
+
+  private
+
+  def clear_timelines(account, target_account)
+    mentions_key = FeedManager.instance.key(:mentions, account.id)
+    home_key     = FeedManager.instance.key(:home, account.id)
+
+    target_account.statuses.select('id').find_each do |status|
+      redis.zrem(mentions_key, status.id)
+      redis.zrem(home_key, status.id)
+    end
+  end
+
+  def clear_notifications(account, target_account)
+    Notification.where(account: account).joins(:follow).where(activity_type: 'Follow', follows: { account_id: target_account.id }).destroy_all
+    Notification.where(account: account).joins(mention: :status).where(activity_type: 'Mention', statuses: { account_id: target_account.id }).destroy_all
+    Notification.where(account: account).joins(:favourite).where(activity_type: 'Favourite', favourites: { account_id: target_account.id }).destroy_all
+    Notification.where(account: account).joins(:status).where(activity_type: 'Status', statuses: { account_id: target_account.id }).destroy_all
+  end
+
+  def redis
+    Redis.current
+  end
+end
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index a8fafe412..9518b1fcf 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -1,15 +1,16 @@
 # frozen_string_literal: true
 
 class BlockDomainService < BaseService
-  def call(domain)
-    DomainBlock.find_or_create_by!(domain: domain)
+  def call(domain, severity)
+    DomainBlock.where(domain: domain).first_or_create!(domain: domain, severity: severity)
 
-    Account.where(domain: domain).find_each do |account|
-      if account.subscribed?
-        account.subscription(api_subscription_url(account.id)).unsubscribe
+    if severity == :silence
+      Account.where(domain: domain).update_all(silenced: true)
+    else
+      Account.where(domain: domain).find_each do |account|
+        account.subscription(api_subscription_url(account.id)).unsubscribe if account.subscribed?
+        SuspendAccountService.new.call(account)
       end
-
-      account.destroy!
     end
   end
 end
diff --git a/app/services/block_service.rb b/app/services/block_service.rb
index b08cf8ca8..e04b6cc39 100644
--- a/app/services/block_service.rb
+++ b/app/services/block_service.rb
@@ -9,32 +9,7 @@ class BlockService < BaseService
 
     block = account.block!(target_account)
 
-    clear_timelines(account, target_account)
-    clear_notifications(account, target_account)
-
+    BlockWorker.perform_async(account.id, target_account.id)
     NotificationWorker.perform_async(block.stream_entry.id, target_account.id) unless target_account.local?
   end
-
-  private
-
-  def clear_timelines(account, target_account)
-    mentions_key = FeedManager.instance.key(:mentions, account.id)
-    home_key     = FeedManager.instance.key(:home, account.id)
-
-    target_account.statuses.select('id').find_each do |status|
-      redis.zrem(mentions_key, status.id)
-      redis.zrem(home_key, status.id)
-    end
-  end
-
-  def clear_notifications(account, target_account)
-    Notification.where(account: account).joins(:follow).where(activity_type: 'Follow', follows: { account_id: target_account.id }).destroy_all
-    Notification.where(account: account).joins(mention: :status).where(activity_type: 'Mention', statuses: { account_id: target_account.id }).destroy_all
-    Notification.where(account: account).joins(:favourite).where(activity_type: 'Favourite', favourites: { account_id: target_account.id }).destroy_all
-    Notification.where(account: account).joins(:status).where(activity_type: 'Status', statuses: { account_id: target_account.id }).destroy_all
-  end
-
-  def redis
-    Redis.current
-  end
 end
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
new file mode 100644
index 000000000..005e5acea
--- /dev/null
+++ b/app/services/fetch_link_card_service.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class FetchLinkCardService < BaseService
+  def call(status)
+    # Get first URL
+    url = URI.extract(status.text).reject { |uri| (uri =~ /\Ahttps?:\/\//).nil? }.first
+
+    return if url.nil?
+
+    response = http_client.get(url)
+
+    return if response.code != 200 || response.mime_type != 'text/html'
+
+    page = Nokogiri::HTML(response.to_s)
+    card = PreviewCard.where(status: status).first_or_initialize(status: status, url: url)
+
+    card.title       = meta_property(page, 'og:title') || page.at_xpath('//title')&.content
+    card.description = meta_property(page, 'og:description') || meta_property(page, 'description')
+    card.image       = URI.parse(meta_property(page, 'og:image')) if meta_property(page, 'og:image')
+
+    return if card.title.blank?
+
+    card.save_with_optional_image!
+  end
+
+  private
+
+  def http_client
+    HTTP.timeout(:per_operation, write: 10, connect: 10, read: 10).follow
+  end
+
+  def meta_property(html, property)
+    html.at_xpath("//meta[@property=\"#{property}\"]")&.attribute('content')&.value || html.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
+  end
+end
diff --git a/app/services/follow_remote_account_service.rb b/app/services/follow_remote_account_service.rb
index f640222b0..b39eafc70 100644
--- a/app/services/follow_remote_account_service.rb
+++ b/app/services/follow_remote_account_service.rb
@@ -14,7 +14,6 @@ class FollowRemoteAccountService < BaseService
     username, domain = uri.split('@')
 
     return Account.find_local(username) if TagManager.instance.local_domain?(domain)
-    return nil if DomainBlock.blocked?(domain)
 
     account = Account.find_remote(username, domain)
     return account unless account.nil?
@@ -36,11 +35,15 @@ class FollowRemoteAccountService < BaseService
 
     Rails.logger.debug "Creating new remote account for #{uri}"
 
+    domain_block = DomainBlock.find_by(domain: domain)
+
     account.remote_url  = data.link('http://schemas.google.com/g/2010#updates-from').href
     account.salmon_url  = data.link('salmon').href
     account.url         = data.link('http://webfinger.net/rel/profile-page').href
     account.public_key  = magic_key_to_pem(data.link('magic-public-key').href)
     account.private_key = nil
+    account.suspended   = true if domain_block && domain_block.suspend?
+    account.silenced    = true if domain_block && domain_block.silence?
 
     xml  = get_feed(account.remote_url)
     hubs = get_hubs(xml)
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 555f01b6d..87c16a621 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -38,7 +38,7 @@ class FollowService < BaseService
       NotificationWorker.perform_async(follow.stream_entry.id, target_account.id)
     end
 
-    FeedManager.instance.merge_into_timeline(target_account, source_account)
+    MergeWorker.perform_async(target_account.id, source_account.id)
     Pubsubhubbub::DistributionWorker.perform_async(follow.stream_entry.id)
 
     follow
diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index 2fb1d3919..1ec36637c 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -6,7 +6,7 @@ class NotifyService < BaseService
     @activity     = activity
     @notification = Notification.new(account: @recipient, activity: @activity)
 
-    return if blocked?
+    return if blocked? || recipient.user.nil?
 
     create_notification
     send_email if email_enabled?
@@ -37,13 +37,13 @@ class NotifyService < BaseService
   end
 
   def blocked?
-    blocked   = @recipient.suspended?                                                                                             # Skip if the recipient account is suspended anyway
-    blocked ||= @recipient.id == @notification.from_account.id                                                                    # Skip for interactions with self
-    blocked ||= @recipient.blocking?(@notification.from_account)                                                                  # Skip for blocked accounts
-    blocked ||= (@notification.from_account.silenced? && !@recipient.following?(@notification.from_account))                      # Hellban
-    blocked ||= (@recipient.user.settings(:interactions).must_be_follower  && !@notification.from_account.following?(@recipient)) # Options
-    blocked ||= (@recipient.user.settings(:interactions).must_be_following && !@recipient.following?(@notification.from_account)) # Options
-    blocked ||= send("blocked_#{@notification.type}?")                                                                            # Type-dependent filters
+    blocked   = @recipient.suspended?                                                                                              # Skip if the recipient account is suspended anyway
+    blocked ||= @recipient.id == @notification.from_account.id                                                                     # Skip for interactions with self
+    blocked ||= @recipient.blocking?(@notification.from_account)                                                                   # Skip for blocked accounts
+    blocked ||= (@notification.from_account.silenced? && !@recipient.following?(@notification.from_account))                       # Hellban
+    blocked ||= (@recipient.user.settings.interactions['must_be_follower']  && !@notification.from_account.following?(@recipient)) # Options
+    blocked ||= (@recipient.user.settings.interactions['must_be_following'] && !@recipient.following?(@notification.from_account)) # Options
+    blocked ||= send("blocked_#{@notification.type}?")                                                                             # Type-dependent filters
     blocked
   end
 
@@ -58,6 +58,6 @@ class NotifyService < BaseService
   end
 
   def email_enabled?
-    @recipient.user.settings(:notification_emails).send(@notification.type)
+    @recipient.user.settings.notification_emails[@notification.type]
   end
 end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 55405c0db..979941c84 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -7,14 +7,24 @@ class PostStatusService < BaseService
   # @param [Status] in_reply_to Optional status to reply to
   # @param [Hash] options
   # @option [Boolean] :sensitive
+  # @option [String] :visibility
+  # @option [String] :spoiler_text
   # @option [Enumerable] :media_ids Optional array of media IDs to attach
+  # @option [Doorkeeper::Application] :application
   # @return [Status]
   def call(account, text, in_reply_to = nil, options = {})
-    status = account.statuses.create!(text: text, thread: in_reply_to, sensitive: options[:sensitive], visibility: options[:visibility])
+    status = account.statuses.create!(text: text,
+                                      thread: in_reply_to,
+                                      sensitive: options[:sensitive],
+                                      spoiler_text: options[:spoiler_text] || '',
+                                      visibility: options[:visibility],
+                                      application: options[:application])
+
     attach_media(status, options[:media_ids])
     process_mentions_service.call(status)
     process_hashtags_service.call(status)
 
+    LinkCrawlWorker.perform_async(status.id)
     DistributionWorker.perform_async(status.id)
     Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)
 
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
index 3860a3504..626534176 100644
--- a/app/services/process_feed_service.rb
+++ b/app/services/process_feed_service.rb
@@ -44,6 +44,8 @@ class ProcessFeedService < BaseService
       Rails.logger.debug "Creating remote status #{id}"
       status = status_from_xml(@xml)
 
+      return if status.nil?
+
       if verb == :share
         original_status = status_from_xml(@xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS))
         status.reblog   = original_status
@@ -59,6 +61,7 @@ class ProcessFeedService < BaseService
       status.save!
 
       NotifyService.new.call(status.reblog.account, status) if status.reblog? && status.reblog.account.local?
+      # LinkCrawlWorker.perform_async(status.reblog? ? status.reblog_of_id : status.id)
       Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
       DistributionWorker.perform_async(status.id)
       status
@@ -100,6 +103,7 @@ class ProcessFeedService < BaseService
         url: url(entry),
         account: account,
         text: content(entry),
+        spoiler_text: content_warning(entry),
         created_at: published(entry)
       )
 
@@ -177,6 +181,8 @@ class ProcessFeedService < BaseService
     end
 
     def media_from_xml(parent, xml)
+      return if DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+
       xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link|
         next unless link['href']
 
@@ -218,6 +224,10 @@ class ProcessFeedService < BaseService
       xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content
     end
 
+    def content_warning(xml = @xml)
+      xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || ''
+    end
+
     def published(xml = @xml)
       xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content
     end
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index ddcc64aa5..617a38159 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -4,7 +4,7 @@ class ProcessHashtagsService < BaseService
   def call(status, tags = [])
     tags = status.text.scan(Tag::HASHTAG_RE).map(&:first) if status.local?
 
-    tags.map { |str| str.mb_chars.downcase }.uniq.each do |tag|
+    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag|
       status.tags << Tag.where(name: tag).first_or_initialize(name: tag)
     end
 
diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb
index 11ec0d2dd..5f91e3127 100644
--- a/app/services/process_interaction_service.rb
+++ b/app/services/process_interaction_service.rb
@@ -17,8 +17,6 @@ class ProcessInteractionService < BaseService
     domain   = Addressable::URI.parse(url).host
     account  = Account.find_by(username: username, domain: domain)
 
-    return if DomainBlock.blocked?(domain)
-
     if account.nil?
       account = follow_remote_account_service.call("#{username}@#{domain}")
     end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index ee42a5df2..72568e702 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -28,7 +28,7 @@ class ProcessMentionsService < BaseService
     status.mentions.each do |mention|
       mentioned_account = mention.account
 
-      next if status.private_visibility? && !mentioned_account.following?(status.account)
+      next if status.private_visibility? && (!mentioned_account.following?(status.account) || !mentioned_account.local?)
 
       if mentioned_account.local?
         NotifyService.new.call(mentioned_account, mention)
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 0cb51eecd..4ea0dbf6c 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -6,7 +6,9 @@ class ReblogService < BaseService
   # @param [Status] reblogged_status Status to be reblogged
   # @return [Status]
   def call(account, reblogged_status)
-    raise Mastodon::NotPermitted if reblogged_status.private_visibility?
+    reblogged_status = reblogged_status.reblog if reblogged_status.reblog?
+
+    raise Mastodon::NotPermitted if reblogged_status.private_visibility? || !reblogged_status.permitted?(account)
 
     reblog = account.statuses.create!(reblog: reblogged_status, text: '')
 
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 836b8fdc5..7aca24d12 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -53,7 +53,12 @@ class RemoveStatusService < BaseService
   end
 
   def unpush(type, receiver, status)
-    redis.zremrangebyscore(FeedManager.instance.key(type, receiver.id), status.id, status.id)
+    if status.reblog?
+      redis.zadd(FeedManager.instance.key(type, receiver.id), status.reblog_of_id, status.reblog_of_id)
+    else
+      redis.zremrangebyscore(FeedManager.instance.key(type, receiver.id), status.id, status.id)
+    end
+
     FeedManager.instance.broadcast(receiver.id, type: 'delete', id: status.id)
   end
 
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 04a086613..8528ef62a 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -18,7 +18,6 @@ class SuspendAccountService < BaseService
 
     @account.media_attachments.destroy_all
     @account.stream_entries.destroy_all
-    @account.mentions.destroy_all
     @account.notifications.destroy_all
     @account.favourites.destroy_all
     @account.active_relationships.destroy_all
diff --git a/app/services/unfollow_service.rb b/app/services/unfollow_service.rb
index 7973a3611..f469793c1 100644
--- a/app/services/unfollow_service.rb
+++ b/app/services/unfollow_service.rb
@@ -7,21 +7,6 @@ class UnfollowService < BaseService
   def call(source_account, target_account)
     follow = source_account.unfollow!(target_account)
     NotificationWorker.perform_async(follow.stream_entry.id, target_account.id) unless target_account.local?
-    unmerge_from_timeline(target_account, source_account)
-  end
-
-  private
-
-  def unmerge_from_timeline(from_account, into_account)
-    timeline_key = FeedManager.instance.key(:home, into_account.id)
-
-    from_account.statuses.select('id').find_each do |status|
-      redis.zrem(timeline_key, status.id)
-      redis.zremrangebyscore(timeline_key, status.id, status.id)
-    end
-  end
-
-  def redis
-    Redis.current
+    UnmergeWorker.perform_async(target_account.id, source_account.id)
   end
 end
diff --git a/app/services/update_remote_profile_service.rb b/app/services/update_remote_profile_service.rb
index d961eda39..ad9c56540 100644
--- a/app/services/update_remote_profile_service.rb
+++ b/app/services/update_remote_profile_service.rb
@@ -8,9 +8,12 @@ class UpdateRemoteProfileService < BaseService
     hub_link   = xml.at_xpath('./xmlns:link[@rel="hub"]', xmlns: TagManager::XMLNS)
 
     unless author_xml.nil?
-      account.display_name      = author_xml.at_xpath('./poco:displayName', poco: TagManager::POCO_XMLNS).content unless author_xml.at_xpath('./poco:displayName', poco: TagManager::POCO_XMLNS).nil?
-      account.note              = author_xml.at_xpath('./poco:note', poco: TagManager::POCO_XMLNS).content unless author_xml.at_xpath('./poco:note', poco: TagManager::POCO_XMLNS).nil?
-      account.avatar_remote_url = author_xml.at_xpath('./xmlns:link[@rel="avatar"]', xmlns: TagManager::XMLNS)['href'] unless author_xml.at_xpath('./xmlns:link[@rel="avatar"]', xmlns: TagManager::XMLNS).nil? || author_xml.at_xpath('./xmlns:link[@rel="avatar"]', xmlns: TagManager::XMLNS)['href'].blank?
+      account.display_name = author_xml.at_xpath('./poco:displayName', poco: TagManager::POCO_XMLNS).content unless author_xml.at_xpath('./poco:displayName', poco: TagManager::POCO_XMLNS).nil?
+      account.note         = author_xml.at_xpath('./poco:note', poco: TagManager::POCO_XMLNS).content unless author_xml.at_xpath('./poco:note', poco: TagManager::POCO_XMLNS).nil?
+
+      unless account.suspended? || DomainBlock.find_by(domain: account.domain)&.reject_media?
+        account.avatar_remote_url = author_xml.at_xpath('./xmlns:link[@rel="avatar"]', xmlns: TagManager::XMLNS)['href'] unless author_xml.at_xpath('./xmlns:link[@rel="avatar"]', xmlns: TagManager::XMLNS).nil? || author_xml.at_xpath('./xmlns:link[@rel="avatar"]', xmlns: TagManager::XMLNS)['href'].blank?
+      end
     end
 
     old_hub_url     = account.hub_url