Merge remote-tracking branch 'origin/master' into gs-master

  Conflicts:
 	.travis.yml
 	Gemfile.lock
 	README.md
 	app/controllers/settings/follower_domains_controller.rb
 	app/controllers/statuses_controller.rb
 	app/javascript/mastodon/locales/ja.json
 	app/lib/feed_manager.rb
 	app/models/media_attachment.rb
 	app/models/mute.rb
 	app/models/status.rb
 	app/services/mute_service.rb
 	app/views/home/index.html.haml
 	app/views/stream_entries/_simple_status.html.haml
 	config/locales/ca.yml
 	config/locales/en.yml
 	config/locales/es.yml
 	config/locales/fr.yml
 	config/locales/nl.yml
 	config/locales/pl.yml
 	config/locales/pt-BR.yml
 	config/themes.yml

14 files changed, 220 insertions, 85 deletions

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 9b00f0f52..5b97a6208 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -80,7 +80,7 @@ class ActivityPub::Activity
 
     # Only continue if the status is supposed to have
     # arrived in real-time
-    return unless @options[:override_timestamps] || status.within_realtime_window?
+    return unless status.within_realtime_window?
 
     distribute_to_followers(status)
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index c8a358195..8840a450c 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -15,7 +15,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       account: @account,
       reblog: original_status,
       uri: @json['id'],
-      created_at: @options[:override_timestamps] ? nil : @json['published'],
+      created_at: @json['published'],
       visibility: original_status.visibility
     )
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 45c0e91cb..edee2691f 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -47,7 +47,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       text: text_from_content || '',
       language: detected_language,
       spoiler_text: @object['summary'] || '',
-      created_at: @options[:override_timestamps] ? nil : @object['published'],
+      created_at: @object['published'],
       reply: @object['inReplyTo'].present?,
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,
@@ -61,12 +61,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if @object['tag'].nil?
 
     as_array(@object['tag']).each do |tag|
-      case tag['type']
-      when 'Hashtag'
+      if equals_or_includes?(tag['type'], 'Hashtag')
         process_hashtag tag, status
-      when 'Mention'
+      elsif equals_or_includes?(tag['type'], 'Mention')
         process_mention tag, status
-      when 'Emoji'
+      elsif equals_or_includes?(tag['type'], 'Emoji')
         process_emoji tag, status
       end
     end
@@ -235,11 +234,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def supported_object_type?
-    SUPPORTED_TYPES.include?(@object['type'])
+    equals_or_includes_any?(@object['type'], SUPPORTED_TYPES)
   end
 
   def converted_object_type?
-    CONVERTED_TYPES.include?(@object['type'])
+    equals_or_includes_any?(@object['type'], CONVERTED_TYPES)
   end
 
   def skip_download?
diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb
index 0134b4015..aa5907f03 100644
--- a/app/lib/activitypub/activity/update.rb
+++ b/app/lib/activitypub/activity/update.rb
@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 class ActivityPub::Activity::Update < ActivityPub::Activity
+  SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
+
   def perform
-    case @object['type']
-    when 'Person'
-      update_account
-    end
+    update_account if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES)
   end
 
   private
diff --git a/app/lib/entity_cache.rb b/app/lib/entity_cache.rb
new file mode 100644
index 000000000..2aa37389c
--- /dev/null
+++ b/app/lib/entity_cache.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+class EntityCache
+  include Singleton
+
+  MAX_EXPIRATION = 7.days.freeze
+
+  def mention(username, domain)
+    Rails.cache.fetch(to_key(:mention, username, domain), expires_in: MAX_EXPIRATION) { Account.select(:username, :domain, :url).find_remote(username, domain) }
+  end
+
+  def emoji(shortcodes, domain)
+    shortcodes   = [shortcodes] unless shortcodes.is_a?(Array)
+    cached       = Rails.cache.read_multi(*shortcodes.map { |shortcode| to_key(:emoji, shortcode, domain) })
+    uncached_ids = []
+
+    shortcodes.each do |shortcode|
+      uncached_ids << shortcode unless cached.key?(to_key(:emoji, shortcode, domain))
+    end
+
+    unless uncached_ids.empty?
+      uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).map { |item| [item.shortcode, item] }.to_h
+      uncached.each_value { |item| Rails.cache.write(to_key(:emoji, item.shortcode, domain), item, expires_in: MAX_EXPIRATION) }
+    end
+
+    shortcodes.map { |shortcode| cached[to_key(:emoji, shortcode, domain)] || uncached[shortcode] }.compact
+  end
+
+  def to_key(type, *ids)
+    "#{type}:#{ids.compact.map(&:downcase).join(':')}"
+  end
+end
diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb
index e88e98eae..01346bfe5 100644
--- a/app/lib/exceptions.rb
+++ b/app/lib/exceptions.rb
@@ -6,6 +6,7 @@ module Mastodon
   class ValidationError < Error; end
   class HostValidationError < ValidationError; end
   class LengthValidationError < ValidationError; end
+  class DimensionsValidationError < ValidationError; end
   class RaceConditionError < Error; end
 
   class UnexpectedResponseError < Error
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 700fd61c4..3a2dcac68 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -145,10 +145,14 @@ class FeedManager
     redis.exists("subscribed:#{timeline_id}")
   end
 
+  def blocks_or_mutes?(receiver_id, account_ids, context)
+    Block.where(account_id: receiver_id, target_account_id: account_ids).any? ||
+      (context == :home ? Mute.where(account_id: receiver_id, target_account_id: account_ids).any? : Mute.where(account_id: receiver_id, target_account_id: account_ids, hide_notifications: true).any?)
+  end
+
   def filter_from_home?(status, receiver_id)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
-
     return true if keyword_filter?(status, receiver_id)
 
     check_for_mutes = [status.account_id]
@@ -158,9 +162,10 @@ class FeedManager
     return true if Mute.where(account_id: receiver_id, target_account_id: check_for_mutes).any?
 
     check_for_blocks = status.mentions.pluck(:account_id)
+    check_for_blocks.concat([status.account_id])
     check_for_blocks.concat([status.reblog.account_id]) if status.reblog?
 
-    return true if Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any?
+    return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home)
 
     if status.reply? && !status.in_reply_to_account_id.nil?                                                                      # Filter out if it's a reply
       should_filter   = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists?         # and I'm not following the person it's a reply to
@@ -184,11 +189,13 @@ class FeedManager
   def filter_from_mentions?(status, receiver_id)
     return true if receiver_id == status.account_id
 
-    check_for_blocks = [status.account_id]
-    check_for_blocks.concat(status.mentions.pluck(:account_id))
+    # This filter is called from NotifyService, but already after the sender of
+    # the notification has been checked for mute/block. Therefore, it's not
+    # necessary to check the author of the toot for mute/block again
+    check_for_blocks = status.mentions.pluck(:account_id)
     check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil?
 
-    should_filter   = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any?                                     # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked
+    should_filter   = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions)                                                         # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted)
     should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them
     should_filter ||= keyword_filter?(status, receiver_id)                                                                               # or if the mention contains a muted keyword
 
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 4124f1660..050c651ee 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -52,12 +52,8 @@ class Formatter
   end
 
   def simplified_format(account, **options)
-    html = if account.local?
-             linkify(account.note)
-           else
-             reformat(account.note)
-           end
-    html = encode_custom_emojis(html, CustomEmoji.from_text(account.note, account.domain)) if options[:custom_emojify]
+    html = account.local? ? linkify(account.note) : reformat(account.note)
+    html = encode_custom_emojis(html, account.emojis) if options[:custom_emojify]
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
@@ -211,7 +207,7 @@ class Formatter
     username, domain = acct.split('@')
 
     domain  = nil if TagManager.instance.local_domain?(domain)
-    account = Account.find_remote(username, domain)
+    account = EntityCache.instance.mention(username, domain)
 
     account ? mention_html(account) : "@#{acct}"
   end
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 6235127b2..a24a0093c 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -39,7 +39,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
         reblog: cached_reblog,
         text: content,
         spoiler_text: content_warning,
-        created_at: @options[:override_timestamps] ? nil : published,
+        created_at: published,
         reply: thread?,
         language: content_language,
         visibility: visibility_scope,
@@ -61,7 +61,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
 
     LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-    DistributionWorker.perform_async(status.id) if @options[:override_timestamps] || status.within_realtime_window?
+    DistributionWorker.perform_async(status.id) if status.within_realtime_window?
 
     status
   end
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
index 055b4649c..7c66f2066 100644
--- a/app/lib/ostatus/atom_serializer.rb
+++ b/app/lib/ostatus/atom_serializer.rb
@@ -364,8 +364,6 @@ class OStatus::AtomSerializer
       append_element(entry, 'category', nil, term: tag.name)
     end
 
-    append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive?
-
     status.media_attachments.each do |media|
       append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
     end
diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb
deleted file mode 100644
index 3bec7211b..000000000
--- a/app/lib/provider_discovery.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-class ProviderDiscovery < OEmbed::ProviderDiscovery
-  class << self
-    def get(url, **options)
-      provider = discover_provider(url, options)
-
-      options.delete(:html)
-
-      provider.get(url, options)
-    end
-
-    def discover_provider(url, **options)
-      format = options[:format]
-
-      html = if options[:html]
-               Nokogiri::HTML(options[:html])
-             else
-               Request.new(:get, url).perform do |res|
-                 raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
-                 Nokogiri::HTML(res.body_with_limit)
-               end
-             end
-
-      if format.nil? || format == :json
-        provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
-        format ||= :json if provider_endpoint
-      end
-
-      if format.nil? || format == :xml
-        provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
-        format ||= :xml if provider_endpoint
-      end
-
-      raise OEmbed::NotFound, url if provider_endpoint.nil?
-      begin
-        provider_endpoint = Addressable::URI.parse(provider_endpoint)
-        provider_endpoint.query = nil
-        provider_endpoint = provider_endpoint.to_s
-      rescue Addressable::URI::InvalidURIError
-        raise OEmbed::NotFound, url
-      end
-
-      OEmbed::Provider.new(provider_endpoint, format)
-    end
-  end
-end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index dca93a6e9..00f94dacf 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -9,11 +9,15 @@ class Request
   include RoutingHelper
 
   def initialize(verb, url, **options)
+    raise ArgumentError if url.blank?
+
     @verb    = verb
     @url     = Addressable::URI.parse(url).normalize
-    @options = options.merge(socket_class: Socket)
+    @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket })
     @headers = {}
 
+    raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
+
     set_common_headers!
     set_digest! if options.key?(:body)
   end
@@ -99,6 +103,14 @@ class Request
     @http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
   end
 
+  def use_proxy?
+    Rails.configuration.x.http_client_proxy.present?
+  end
+
+  def block_hidden_service?
+    !Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(@url.host)
+  end
+
   module ClientLimit
     def body_with_limit(limit = 1.megabyte)
       raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
@@ -129,6 +141,7 @@ class Request
   class Socket < TCPSocket
     class << self
       def open(host, *args)
+        return super host, *args if thru_hidden_service? host
         outer_e = nil
         Addrinfo.foreach(host, nil, nil, :SOCK_STREAM) do |address|
           begin
@@ -142,6 +155,10 @@ class Request
       end
 
       alias new open
+
+      def thru_hidden_service?(host)
+        Rails.configuration.x.hidden_service_via_transparent_proxy && /\.(onion|i2p)$/.match(host)
+      end
     end
   end
 
diff --git a/app/lib/rss_builder.rb b/app/lib/rss_builder.rb
new file mode 100644
index 000000000..63ddba2e8
--- /dev/null
+++ b/app/lib/rss_builder.rb
@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+class RSSBuilder
+  class ItemBuilder
+    def initialize
+      @item = Ox::Element.new('item')
+    end
+
+    def title(str)
+      @item << (Ox::Element.new('title') << str)
+
+      self
+    end
+
+    def link(str)
+      @item << Ox::Element.new('guid').tap do |guid|
+        guid['isPermalink'] = 'true'
+        guid << str
+      end
+
+      @item << (Ox::Element.new('link') << str)
+
+      self
+    end
+
+    def pub_date(date)
+      @item << (Ox::Element.new('pubDate') << date.to_formatted_s(:rfc822))
+
+      self
+    end
+
+    def description(str)
+      @item << (Ox::Element.new('description') << str)
+
+      self
+    end
+
+    def enclosure(url, type, size)
+      @item << Ox::Element.new('enclosure').tap do |enclosure|
+        enclosure['url']    = url
+        enclosure['length'] = size
+        enclosure['type']   = type
+      end
+
+      self
+    end
+
+    def to_element
+      @item
+    end
+  end
+
+  def initialize
+    @document = Ox::Document.new(version: '1.0')
+    @channel  = Ox::Element.new('channel')
+
+    @document << (rss << @channel)
+  end
+
+  def title(str)
+    @channel << (Ox::Element.new('title') << str)
+
+    self
+  end
+
+  def link(str)
+    @channel << (Ox::Element.new('link') << str)
+
+    self
+  end
+
+  def image(str)
+    @channel << Ox::Element.new('image').tap do |image|
+      image << (Ox::Element.new('url') << str)
+      image << (Ox::Element.new('title') << '')
+      image << (Ox::Element.new('link') << '')
+    end
+
+    @channel << (Ox::Element.new('webfeeds:icon') << str)
+
+    self
+  end
+
+  def cover(str)
+    @channel << Ox::Element.new('webfeeds:cover').tap do |cover|
+      cover['image'] = str
+    end
+
+    self
+  end
+
+  def logo(str)
+    @channel << (Ox::Element.new('webfeeds:logo') << str)
+
+    self
+  end
+
+  def accent_color(str)
+    @channel << (Ox::Element.new('webfeeds:accentColor') << str)
+
+    self
+  end
+
+  def description(str)
+    @channel << (Ox::Element.new('description') << str)
+
+    self
+  end
+
+  def item
+    @channel << ItemBuilder.new.tap do |item|
+      yield item
+    end.to_element
+
+    self
+  end
+
+  def to_xml
+    ('<?xml version="1.0" encoding="UTF-8"?>' + Ox.dump(@document, effort: :tolerant)).force_encoding('UTF-8')
+  end
+
+  private
+
+  def rss
+    Ox::Element.new('rss').tap do |rss|
+      rss['version']        = '2.0'
+      rss['xmlns:webfeeds'] = 'http://webfeeds.org/rss/1.0'
+    end
+  end
+end
diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb
index 41d4381e5..b6c80b801 100644
--- a/app/lib/status_filter.rb
+++ b/app/lib/status_filter.rb
@@ -3,9 +3,10 @@
 class StatusFilter
   attr_reader :status, :account
 
-  def initialize(status, account)
-    @status = status
-    @account = account
+  def initialize(status, account, preloaded_relations = {})
+    @status              = status
+    @account             = account
+    @preloaded_relations = preloaded_relations
   end
 
   def filtered?
@@ -24,15 +25,15 @@ class StatusFilter
   end
 
   def blocking_account?
-    account.blocking? status.account_id
+    @preloaded_relations[:blocking] ? @preloaded_relations[:blocking][status.account_id] : account.blocking?(status.account_id)
   end
 
   def blocking_domain?
-    account.domain_blocking? status.account_domain
+    @preloaded_relations[:domain_blocking_by_domain] ? @preloaded_relations[:domain_blocking_by_domain][status.account_domain] : account.domain_blocking?(status.account_domain)
   end
 
   def muting_account?
-    account.muting? status.account_id
+    @preloaded_relations[:muting] ? @preloaded_relations[:muting][status.account_id] : account.muting?(status.account_id)
   end
 
   def silenced_account?
@@ -44,7 +45,7 @@ class StatusFilter
   end
 
   def account_following_status_account?
-    account&.following? status.account_id
+    @preloaded_relations[:following] ? @preloaded_relations[:following][status.account_id] : account&.following?(status.account_id)
   end
 
   def blocked_by_policy?
@@ -52,6 +53,6 @@ class StatusFilter
   end
 
   def policy_allows_show?
-    StatusPolicy.new(account, status).show?
+    StatusPolicy.new(account, status, @preloaded_relations).show?
   end
 end