Merge remote-tracking branch 'origin/master' into gs-master

  Conflicts:
 	.travis.yml
 	Gemfile.lock
 	README.md
 	app/controllers/settings/follower_domains_controller.rb
 	app/controllers/statuses_controller.rb
 	app/javascript/mastodon/locales/ja.json
 	app/lib/feed_manager.rb
 	app/models/media_attachment.rb
 	app/models/mute.rb
 	app/models/status.rb
 	app/services/mute_service.rb
 	app/views/home/index.html.haml
 	app/views/stream_entries/_simple_status.html.haml
 	config/locales/ca.yml
 	config/locales/en.yml
 	config/locales/es.yml
 	config/locales/fr.yml
 	config/locales/nl.yml
 	config/locales/pl.yml
 	config/locales/pt-BR.yml
 	config/themes.yml

16 files changed, 135 insertions, 61 deletions

diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index 3860a9cbd..7edbd9b47 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -65,9 +65,9 @@ class AccountSearchService < BaseService
   def exact_match
     @_exact_match ||= begin
       if domain_is_local?
-        search_from.find_local(query_username)
+        search_from.without_suspended.find_local(query_username)
       else
-        search_from.find_remote(query_username, query_domain)
+        search_from.without_suspended.find_remote(query_username, query_domain)
       end
     end
   end
diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index 40714e980..6a137b520 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -4,6 +4,8 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
   include JsonLdHelper
 
   def call(account)
+    return if account.featured_collection_url.blank?
+
     @account = account
     @json    = fetch_resource(@account.featured_collection_url, true)
 
diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb
index 5024853ca..867e70876 100644
--- a/app/services/activitypub/fetch_remote_account_service.rb
+++ b/app/services/activitypub/fetch_remote_account_service.rb
@@ -56,6 +56,6 @@ class ActivityPub::FetchRemoteAccountService < BaseService
   end
 
   def expected_type?
-    SUPPORTED_TYPES.include?(@json['type'])
+    equals_or_includes_any?(@json['type'], SUPPORTED_TYPES)
   end
 end
diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb
index 41837d462..505baccd4 100644
--- a/app/services/activitypub/fetch_remote_key_service.rb
+++ b/app/services/activitypub/fetch_remote_key_service.rb
@@ -43,7 +43,7 @@ class ActivityPub::FetchRemoteKeyService < BaseService
   end
 
   def person?
-    ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@json['type'])
+    equals_or_includes_any?(@json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES)
   end
 
   def public_key?
@@ -55,6 +55,6 @@ class ActivityPub::FetchRemoteKeyService < BaseService
   end
 
   def confirmed_owner?
-    ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@owner['type']) && value_or_id(@owner['publicKey']) == @json['id']
+    equals_or_includes_any?(@owner['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && value_or_id(@owner['publicKey']) == @json['id']
   end
 end
diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index 503c175d8..930fbad1f 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -42,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   end
 
   def expected_type?
-    (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? @json['type']
+    equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
   end
 
   def needs_update(actor)
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index da32f9615..f67ebb443 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -201,10 +201,7 @@ class ActivityPub::ProcessAccountService < BaseService
     return if @json['tag'].blank?
 
     as_array(@json['tag']).each do |tag|
-      case tag['type']
-      when 'Emoji'
-        process_emoji tag
-      end
+      process_emoji tag if equals_or_includes?(tag['type'], 'Emoji')
     end
   end
 
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 0f77556dc..510b80c82 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status
diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb
index 44df3ed13..bc2d1547a 100644
--- a/app/services/favourite_service.rb
+++ b/app/services/favourite_service.rb
@@ -8,7 +8,7 @@ class FavouriteService < BaseService
   # @param [Status] status
   # @return [Favourite]
   def call(account, status)
-    authorize_with account, status, :show?
+    authorize_with account, status, :favourite?
 
     favourite = Favourite.find_by(account: account, status: status)
 
diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb
index 0444baf74..550e75f33 100644
--- a/app/services/fetch_atom_service.rb
+++ b/app/services/fetch_atom_service.rb
@@ -42,7 +42,7 @@ class FetchAtomService < BaseService
     elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type)
       body = response.body_with_limit
       json = body_to_json(body)
-      if supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) && json['inbox'].present?
+      if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present?
         [json['id'], { prefetched_body: body, id: true }, :activitypub]
       elsif supported_context?(json) && expected_type?(json)
         [json['id'], { prefetched_body: body, id: true }, :activitypub]
@@ -62,7 +62,7 @@ class FetchAtomService < BaseService
   end
 
   def expected_type?(json)
-    (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? json['type']
+    equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
   end
 
   def process_html(response)
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index d5920a417..77d4aa538 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_oembed
-    embed = OEmbed::Providers.get(@url, html: @html)
+    embed = FetchOEmbedService.new.call(@url, html: @html)
 
-    return false unless embed.respond_to?(:type)
+    return false if embed.nil?
 
-    @card.type          = embed.type
-    @card.title         = embed.respond_to?(:title)         ? embed.title         : ''
-    @card.author_name   = embed.respond_to?(:author_name)   ? embed.author_name   : ''
-    @card.author_url    = embed.respond_to?(:author_url)    ? embed.author_url    : ''
-    @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : ''
-    @card.provider_url  = embed.respond_to?(:provider_url)  ? embed.provider_url  : ''
+    @card.type          = embed[:type]
+    @card.title         = embed[:title]         || ''
+    @card.author_name   = embed[:author_name]   || ''
+    @card.author_url    = embed[:author_url]    || ''
+    @card.provider_name = embed[:provider_name] || ''
+    @card.provider_url  = embed[:provider_url]  || ''
     @card.width         = 0
     @card.height        = 0
 
     case @card.type
     when 'link'
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'photo'
-      return false unless embed.respond_to?(:url)
+      return false if embed[:url].blank?
 
-      @card.embed_url        = embed.url
-      @card.image_remote_url = embed.url
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
+      @card.embed_url        = embed[:url]
+      @card.image_remote_url = embed[:url]
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
     when 'video'
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
-      @card.html             = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED)
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
+      @card.html             = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'rich'
       # Most providers rely on <script> tags, which is a no-no
       return false
     end
 
     @card.save_with_optional_image!
-  rescue OEmbed::NotFound
-    false
   end
 
   def attempt_opengraph
diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb
new file mode 100644
index 000000000..998228517
--- /dev/null
+++ b/app/services/fetch_oembed_service.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class FetchOEmbedService
+  attr_reader :url, :options, :format, :endpoint_url
+
+  def call(url, options = {})
+    @url     = url
+    @options = options
+
+    discover_endpoint!
+    fetch!
+  end
+
+  private
+
+  def discover_endpoint!
+    return if html.nil?
+
+    @format = @options[:format]
+    page    = Nokogiri::HTML(html)
+
+    if @format.nil? || @format == :json
+      @endpoint_url ||= page.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
+      @format       ||= :json if @endpoint_url
+    end
+
+    if @format.nil? || @format == :xml
+      @endpoint_url ||= page.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
+      @format       ||= :xml if @endpoint_url
+    end
+
+    return if @endpoint_url.blank?
+
+    @endpoint_url = Addressable::URI.parse(@endpoint_url).to_s
+  rescue Addressable::URI::InvalidURIError
+    @endpoint_url = nil
+  end
+
+  def fetch!
+    return if @endpoint_url.blank?
+
+    body = Request.new(:get, @endpoint_url).perform do |res|
+      res.code != 200 ? nil : res.body_with_limit
+    end
+
+    validate(parse_for_format(body)) unless body.nil?
+  rescue Oj::ParseError, Ox::ParseError
+    nil
+  end
+
+  def parse_for_format(body)
+    case @format
+    when :json
+      Oj.load(body, mode: :strict)&.with_indifferent_access
+    when :xml
+      Ox.load(body, mode: :hash_no_attrs)&.with_indifferent_access&.dig(:oembed)
+    end
+  end
+
+  def validate(oembed)
+    oembed if oembed[:version] == '1.0' && oembed[:type].present?
+  end
+
+  def html
+    return @html if defined?(@html)
+
+    @html = @options[:html] || Request.new(:get, @url).perform do |res|
+      res.code != 200 || res.mime_type != 'text/html' ? nil : res.body_with_limit
+    end
+  end
+end
diff --git a/app/services/mute_service.rb b/app/services/mute_service.rb
index 547b2efa1..c6122a152 100644
--- a/app/services/mute_service.rb
+++ b/app/services/mute_service.rb
@@ -3,8 +3,13 @@
 class MuteService < BaseService
   def call(account, target_account, notifications: nil)
     return if account.id == target_account.id
+
     mute = account.mute!(target_account, notifications: notifications)
-    BlockWorker.perform_async(account.id, target_account.id)
+    if mute.hide_notifications?
+      BlockWorker.perform_async(account.id, target_account.id)
+    else
+      FeedManager.instance.clear_from_timeline(account, target_account)
+    end
     mute
   end
 end
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index 990e01a4b..5b45c865f 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -7,7 +7,5 @@ class ProcessHashtagsService < BaseService
     tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag|
       status.tags << Tag.where(name: tag).first_or_initialize(name: tag)
     end
-
-    status.update(sensitive: true) if tags.include?('nsfw')
   end
 end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index dc8df4a9a..2ed6698cf 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -10,55 +10,61 @@ class ProcessMentionsService < BaseService
   def call(status)
     return unless status.local?
 
+    @status  = status
+    mentions = []
+
     status.text = status.text.gsub(Account::MENTION_RE) do |match|
-      username, domain  = $1.split('@')
+      username, domain  = Regexp.last_match(1).split('@')
       mentioned_account = Account.find_remote(username, domain)
 
-      if mention_undeliverable?(status, mentioned_account)
+      if mention_undeliverable?(mentioned_account)
         begin
-          mentioned_account = resolve_account_service.call($1)
+          mentioned_account = resolve_account_service.call(Regexp.last_match(1))
         rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError
           mentioned_account = nil
         end
       end
 
-      next match if mention_undeliverable?(status, mentioned_account)
+      next match if mention_undeliverable?(mentioned_account)
+
+      mentions << mentioned_account.mentions.where(status: status).first_or_create(status: status)
 
-      mentioned_account.mentions.where(status: status).first_or_create(status: status)
       "@#{mentioned_account.acct}"
     end
 
     status.save!
 
-    status.mentions.includes(:account).each do |mention|
-      create_notification(status, mention)
-    end
+    mentions.each { |mention| create_notification(mention) }
   end
 
   private
 
-  def mention_undeliverable?(status, mentioned_account)
-    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus? && status.stream_entry.hidden?)
+  def mention_undeliverable?(mentioned_account)
+    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus? && @status.stream_entry.hidden?)
   end
 
-  def create_notification(status, mention)
+  def create_notification(mention)
     mentioned_account = mention.account
 
     if mentioned_account.local?
-      NotifyService.new.call(mentioned_account, mention)
-    elsif mentioned_account.ostatus? && !status.stream_entry.hidden?
-      NotificationWorker.perform_async(stream_entry_to_xml(status.stream_entry), status.account_id, mentioned_account.id)
+      LocalNotificationWorker.perform_async(mention.id)
+    elsif mentioned_account.ostatus? && !@status.stream_entry.hidden?
+      NotificationWorker.perform_async(ostatus_xml, @status.account_id, mentioned_account.id)
     elsif mentioned_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(mention.status), mention.status.account_id, mentioned_account.inbox_url)
+      ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url)
     end
   end
 
-  def build_json(status)
-    Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
-      status,
+  def ostatus_xml
+    @ostatus_xml ||= stream_entry_to_xml(@status.stream_entry)
+  end
+
+  def activitypub_json
+    @activitypub_json ||= Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
+      @status,
       serializer: ActivityPub::ActivitySerializer,
       adapter: ActivityPub::Adapter
-    ).as_json).sign!(status.account))
+    ).as_json).sign!(@status.account))
   end
 
   def resolve_account_service
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 8cba88f01..de8d1151d 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -189,7 +189,7 @@ class ResolveAccountService < BaseService
     return @actor_json if defined?(@actor_json)
 
     json        = fetch_resource(actor_url, false)
-    @actor_json = supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) ? json : nil
+    @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil
   end
 
   def atom
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index c19b568cb..a068c1ed8 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -16,10 +16,9 @@ class ResolveURLService < BaseService
   private
 
   def process_url
-    case type
-    when 'Application', 'Group', 'Organization', 'Person', 'Service'
+    if equals_or_includes_any?(type, %w(Application Group Organization Person Service))
       FetchRemoteAccountService.new.call(atom_url, body, protocol)
-    when 'Note', 'Article', 'Image', 'Video'
+    elsif equals_or_includes_any?(type, %w(Note Article Image Video))
       FetchRemoteStatusService.new.call(atom_url, body, protocol)
     end
   end