From 790d3bc6370f1baf0d00ccf89e81387204c65194 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 11 Oct 2018 00:50:18 +0200
Subject: Move network calls out of transaction in ActivityPub handler (#8951)

Mention and emoji code may perform network calls, but does not need
to do that inside the database transaction. This may improve availability
of database connections when using pgBouncer in transaction mode.
---
 app/lib/activitypub/activity/create.rb | 85 +++++++++++++++++++++-------------
 1 file changed, 52 insertions(+), 33 deletions(-)

(limited to 'app/lib')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 978289788..b889461cb 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -22,12 +22,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   private
 
   def process_status
-    status_params = process_status_params
+    @tags     = []
+    @mentions = []
+    @params   = {}
 
-    ApplicationRecord.transaction do
-      @status = Status.create!(status_params)
+    process_status_params
+    process_tags
 
-      process_tags(@status)
+    ApplicationRecord.transaction do
+      @status = Status.create!(@params)
+      attach_tags(@status)
     end
 
     resolve_thread(@status)
@@ -42,62 +46,77 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_status_params
-    {
-      uri: @object['id'],
-      url: object_url || @object['id'],
-      account: @account,
-      text: text_from_content || '',
-      language: detected_language,
-      spoiler_text: text_from_summary || '',
-      created_at: @object['published'],
-      override_timestamps: @options[:override_timestamps],
-      reply: @object['inReplyTo'].present?,
-      sensitive: @object['sensitive'] || false,
-      visibility: visibility_from_audience,
-      thread: replied_to_status,
-      conversation: conversation_from_uri(@object['conversation']),
-      media_attachment_ids: process_attachments.take(4).map(&:id),
-    }
-  end
-
-  def process_tags(status)
+    @params = begin
+      {
+        uri: @object['id'],
+        url: object_url || @object['id'],
+        account: @account,
+        text: text_from_content || '',
+        language: detected_language,
+        spoiler_text: text_from_summary || '',
+        created_at: @object['published'],
+        override_timestamps: @options[:override_timestamps],
+        reply: @object['inReplyTo'].present?,
+        sensitive: @object['sensitive'] || false,
+        visibility: visibility_from_audience,
+        thread: replied_to_status,
+        conversation: conversation_from_uri(@object['conversation']),
+        media_attachment_ids: process_attachments.take(4).map(&:id),
+      }
+    end
+  end
+
+  def attach_tags(status)
+    @tags.each do |tag|
+      status.tags << tag
+      TrendingTags.record_use!(hashtag, status.account, status.created_at) if status.public_visibility?
+    end
+
+    @mentions.each do |mention|
+      mention.status = status
+      mention.save
+    end
+  end
+
+  def process_tags
     return if @object['tag'].nil?
 
     as_array(@object['tag']).each do |tag|
       if equals_or_includes?(tag['type'], 'Hashtag')
-        process_hashtag tag, status
+        process_hashtag tag
       elsif equals_or_includes?(tag['type'], 'Mention')
-        process_mention tag, status
+        process_mention tag
       elsif equals_or_includes?(tag['type'], 'Emoji')
-        process_emoji tag, status
+        process_emoji tag
       end
     end
   end
 
-  def process_hashtag(tag, status)
+  def process_hashtag(tag)
     return if tag['name'].blank?
 
     hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
     hashtag = Tag.where(name: hashtag).first_or_create(name: hashtag)
 
-    return if status.tags.include?(hashtag)
+    return if @tags.include?(hashtag)
 
-    status.tags << hashtag
-    TrendingTags.record_use!(hashtag, status.account, status.created_at) if status.public_visibility?
+    @tags << hashtag
   rescue ActiveRecord::RecordInvalid
     nil
   end
 
-  def process_mention(tag, status)
+  def process_mention(tag)
     return if tag['href'].blank?
 
     account = account_from_uri(tag['href'])
     account = ::FetchRemoteAccountService.new.call(tag['href'], id: false) if account.nil?
+
     return if account.nil?
-    account.mentions.create(status: status)
+
+    @mentions << Mention.new(account: account)
   end
 
-  def process_emoji(tag, _status)
+  def process_emoji(tag)
     return if skip_download?
     return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
 
-- 
cgit 


From 61d44dd11ffbe965319680cbfb9e787811c11b94 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 11 Oct 2018 02:10:15 +0200
Subject: Fix typo in ActivityPub Create handler (#8952)

Regression from #8951
---
 app/lib/activitypub/activity/create.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/lib')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index b889461cb..73475bf02 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -69,7 +69,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
-      TrendingTags.record_use!(hashtag, status.account, status.created_at) if status.public_visibility?
+      TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
     end
 
     @mentions.each do |mention|
-- 
cgit 


From 21ad21cb507d7a5f48ef8ee726b2f9308052aa9d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 12 Oct 2018 00:15:55 +0200
Subject: Improve signature verification safeguards (#8959)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Downcase signed_headers string before building the signed string

The HTTP Signatures draft does not mandate the “headers” field to be downcased,
but mandates the header field names to be downcased in the signed string, which
means that prior to this patch, Mastodon could fail to process signatures from
some compliant clients. It also means that it would not actually check the
Digest of non-compliant clients that wouldn't use a lowercased Digest field
name.

Thankfully, I don't know of any such client.

* Revert "Remove dead code (#8919)"

This reverts commit a00ce8c92c06f42109aad5cfe65d46862cf037bb.

* Restore time window checking, change it to 12 hours

By checking the Date header, we can prevent replaying old vulnerable
signatures. The focus is to prevent replaying old vulnerable requests
from software that has been fixed in the meantime, so a somewhat long
window should be fine and accounts for timezone misconfiguration.

* Escape users' URLs when formatting them

Fixes possible HTML injection

* Escape all string interpolations in Formatter class

Slightly improve performance by reducing class allocations
from repeated Formatter#encode calls

* Fix code style issues
---
 app/controllers/concerns/signature_verification.rb | 18 +++++++++++++++-
 app/lib/formatter.rb                               | 16 +++++++++------
 .../concerns/signature_verification_spec.rb        | 24 ++++++++++++++++++++++
 3 files changed, 51 insertions(+), 7 deletions(-)

(limited to 'app/lib')

diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb
index 5f95fa346..e5d5e2ca6 100644
--- a/app/controllers/concerns/signature_verification.rb
+++ b/app/controllers/concerns/signature_verification.rb
@@ -22,6 +22,12 @@ module SignatureVerification
       return
     end
 
+    if request.headers['Date'].present? && !matches_time_window?
+      @signature_verification_failure_reason = 'Signed request date outside acceptable time window'
+      @signed_request_account = nil
+      return
+    end
+
     raw_signature    = request.headers['Signature']
     signature_params = {}
 
@@ -76,7 +82,7 @@ module SignatureVerification
   def build_signed_string(signed_headers)
     signed_headers = 'date' if signed_headers.blank?
 
-    signed_headers.split(' ').map do |signed_header|
+    signed_headers.downcase.split(' ').map do |signed_header|
       if signed_header == Request::REQUEST_TARGET
         "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
       elsif signed_header == 'digest'
@@ -87,6 +93,16 @@ module SignatureVerification
     end.join("\n")
   end
 
+  def matches_time_window?
+    begin
+      time_sent = Time.httpdate(request.headers['Date'])
+    rescue ArgumentError
+      return false
+    end
+
+    (Time.now.utc - time_sent).abs <= 12.hours
+  end
+
   def body_digest
     "SHA-256=#{Digest::SHA256.base64digest(request_body)}"
   end
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 8b694536c..35d5a09b7 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -90,8 +90,12 @@ class Formatter
 
   private
 
+  def html_entities
+    @html_entities ||= HTMLEntities.new
+  end
+
   def encode(html)
-    HTMLEntities.new.encode(html)
+    html_entities.encode(html)
   end
 
   def encode_and_link_urls(html, accounts = nil, options = {})
@@ -143,7 +147,7 @@ class Formatter
         emoji     = emoji_map[shortcode]
 
         if emoji
-          replacement = "<img draggable=\"false\" class=\"emojione\" alt=\":#{shortcode}:\" title=\":#{shortcode}:\" src=\"#{emoji}\" />"
+          replacement = "<img draggable=\"false\" class=\"emojione\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(emoji)}\" />"
           before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
           html        = before_html + replacement + html[i + 1..-1]
           i          += replacement.size - (shortcode.size + 2) - 1
@@ -212,7 +216,7 @@ class Formatter
     return link_to_account(acct) unless linkable_accounts
 
     account = linkable_accounts.find { |item| TagManager.instance.same_acct?(item.acct, acct) }
-    account ? mention_html(account) : "@#{acct}"
+    account ? mention_html(account) : "@#{encode(acct)}"
   end
 
   def link_to_account(acct)
@@ -221,7 +225,7 @@ class Formatter
     domain  = nil if TagManager.instance.local_domain?(domain)
     account = EntityCache.instance.mention(username, domain)
 
-    account ? mention_html(account) : "@#{acct}"
+    account ? mention_html(account) : "@#{encode(acct)}"
   end
 
   def link_to_hashtag(entity)
@@ -239,10 +243,10 @@ class Formatter
   end
 
   def hashtag_html(tag)
-    "<a href=\"#{tag_url(tag.downcase)}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{tag}</span></a>"
+    "<a href=\"#{encode(tag_url(tag.downcase))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>"
   end
 
   def mention_html(account)
-    "<span class=\"h-card\"><a href=\"#{TagManager.instance.url_for(account)}\" class=\"u-url mention\">@<span>#{account.username}</span></a></span>"
+    "<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
   end
 end
diff --git a/spec/controllers/concerns/signature_verification_spec.rb b/spec/controllers/concerns/signature_verification_spec.rb
index 3daf1fc4e..720690097 100644
--- a/spec/controllers/concerns/signature_verification_spec.rb
+++ b/spec/controllers/concerns/signature_verification_spec.rb
@@ -73,6 +73,30 @@ describe ApplicationController, type: :controller do
       end
     end
 
+    context 'with request older than a day' do
+      before do
+        get :success
+
+        fake_request = Request.new(:get, request.url)
+        fake_request.add_headers({ 'Date' => 2.days.ago.utc.httpdate })
+        fake_request.on_behalf_of(author)
+
+        request.headers.merge!(fake_request.headers)
+      end
+
+      describe '#signed_request?' do
+        it 'returns true' do
+          expect(controller.signed_request?).to be true
+        end
+      end
+
+      describe '#signed_request_account' do
+        it 'returns nil' do
+          expect(controller.signed_request_account).to be_nil
+        end
+      end
+    end
+
     context 'with body' do
       before do
         post :success, body: 'Hello world'
-- 
cgit 


From ddd30f331c7a2af38176d72d9ce2265068984bed Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 17 Oct 2018 17:13:04 +0200
Subject: Improve support for aspects/circles (#8950)

* Add silent column to mentions

* Save silent mentions in ActivityPub Create handler and optimize it

Move networking calls out of the database transaction

* Add "limited" visibility level masked as "private" in the API

Unlike DMs, limited statuses are pushed into home feeds. The access
control rules between direct and limited statuses is almost the same,
except for counter and conversation logic

* Ensure silent column is non-null, add spec

* Ensure filters don't check silent mentions for blocks/mutes

As those are "this person is also allowed to see" rather than "this
person is involved", therefore does not warrant filtering

* Clean up code

* Use Status#active_mentions to limit returned mentions

* Fix code style issues

* Use Status#active_mentions in Notification

And remove stream_entry eager-loading from Notification
---
 app/lib/activitypub/activity.rb                    |  2 +-
 app/lib/activitypub/activity/create.rb             | 24 +++++++++++++++++-
 app/lib/activitypub/tag_manager.rb                 |  6 ++---
 app/lib/feed_manager.rb                            |  8 +++---
 app/lib/formatter.rb                               |  2 +-
 app/lib/ostatus/atom_serializer.rb                 |  2 +-
 app/models/account_conversation.rb                 |  2 +-
 app/models/mention.rb                              |  8 ++++++
 app/models/notification.rb                         |  2 +-
 app/models/status.rb                               | 15 +++++++----
 app/models/stream_entry.rb                         |  2 +-
 app/policies/status_policy.rb                      |  8 +++---
 app/serializers/activitypub/note_serializer.rb     |  2 +-
 app/serializers/rest/status_serializer.rb          | 13 +++++++++-
 app/services/batched_remove_status_service.rb      |  2 +-
 app/services/fan_out_on_write_service.rb           | 12 +++++++++
 app/services/remove_status_service.rb              |  2 +-
 .../stream_entries/_detailed_status.html.haml      |  2 +-
 app/workers/activitypub/distribution_worker.rb     |  2 +-
 .../activitypub/reply_distribution_worker.rb       |  6 +----
 .../20181010141500_add_silent_to_mentions.rb       | 23 +++++++++++++++++
 db/schema.rb                                       |  3 ++-
 spec/lib/activitypub/activity/create_spec.rb       | 29 ++++++++++++++++++++++
 23 files changed, 142 insertions(+), 35 deletions(-)
 create mode 100644 db/migrate/20181010141500_add_silent_to_mentions.rb

(limited to 'app/lib')

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 3a39b723e..999954cb5 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -96,7 +96,7 @@ class ActivityPub::Activity
   end
 
   def notify_about_mentions(status)
-    status.mentions.includes(:account).each do |mention|
+    status.active_mentions.includes(:account).each do |mention|
       next unless mention.account.local? && audience_includes?(mention.account)
       NotifyService.new.call(mention.account, mention)
     end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 73475bf02..7e6702a63 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -28,6 +28,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     process_status_params
     process_tags
+    process_audience
 
     ApplicationRecord.transaction do
       @status = Status.create!(@params)
@@ -66,6 +67,27 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     end
   end
 
+  def process_audience
+    (as_array(@object['to']) + as_array(@object['cc'])).uniq.each do |audience|
+      next if audience == ActivityPub::TagManager::COLLECTIONS[:public]
+
+      # Unlike with tags, there is no point in resolving accounts we don't already
+      # know here, because silent mentions would only be used for local access
+      # control anyway
+      account = account_from_uri(audience)
+
+      next if account.nil? || @mentions.any? { |mention| mention.account_id == account.id }
+
+      @mentions << Mention.new(account: account, silent: true)
+
+      # If there is at least one silent mention, then the status can be considered
+      # as a limited-audience status, and not strictly a direct message
+      next unless @params[:visibility] == :direct
+
+      @params[:visibility] = :limited
+    end
+  end
+
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
@@ -113,7 +135,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     return if account.nil?
 
-    @mentions << Mention.new(account: account)
+    @mentions << Mention.new(account: account, silent: false)
   end
 
   def process_emoji(tag)
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 95d1cf9f3..be3a562d0 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -58,8 +58,8 @@ class ActivityPub::TagManager
       [COLLECTIONS[:public]]
     when 'unlisted', 'private'
       [account_followers_url(status.account)]
-    when 'direct'
-      status.mentions.map { |mention| uri_for(mention.account) }
+    when 'direct', 'limited'
+      status.active_mentions.map { |mention| uri_for(mention.account) }
     end
   end
 
@@ -80,7 +80,7 @@ class ActivityPub::TagManager
       cc << COLLECTIONS[:public]
     end
 
-    cc.concat(status.mentions.map { |mention| uri_for(mention.account) }) unless status.direct_visibility?
+    cc.concat(status.active_mentions.map { |mention| uri_for(mention.account) }) unless status.direct_visibility? || status.limited_visibility?
 
     cc
   end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index b10e5dd24..3d7db2721 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -88,7 +88,7 @@ class FeedManager
     end
 
     query.each do |status|
-      next if status.direct_visibility? || filter?(:home, status, into_account)
+      next if status.direct_visibility? || status.limited_visibility? || filter?(:home, status, into_account)
       add_to_feed(:home, into_account.id, status)
     end
 
@@ -156,12 +156,12 @@ class FeedManager
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
     return true  if phrase_filtered?(status, receiver_id, :home)
 
-    check_for_blocks = status.mentions.pluck(:account_id)
+    check_for_blocks = status.active_mentions.pluck(:account_id)
     check_for_blocks.concat([status.account_id])
 
     if status.reblog?
       check_for_blocks.concat([status.reblog.account_id])
-      check_for_blocks.concat(status.reblog.mentions.pluck(:account_id))
+      check_for_blocks.concat(status.reblog.active_mentions.pluck(:account_id))
     end
 
     return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home)
@@ -188,7 +188,7 @@ class FeedManager
     # This filter is called from NotifyService, but already after the sender of
     # the notification has been checked for mute/block. Therefore, it's not
     # necessary to check the author of the toot for mute/block again
-    check_for_blocks = status.mentions.pluck(:account_id)
+    check_for_blocks = status.active_mentions.pluck(:account_id)
     check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil?
 
     should_filter   = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions)                                                         # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted)
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 35d5a09b7..d13884ec8 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -27,7 +27,7 @@ class Formatter
       return html.html_safe # rubocop:disable Rails/OutputSafety
     end
 
-    linkable_accounts = status.mentions.map(&:account)
+    linkable_accounts = status.active_mentions.map(&:account)
     linkable_accounts << status.account
 
     html = raw_content
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
index 1a0a635b3..7a181fb40 100644
--- a/app/lib/ostatus/atom_serializer.rb
+++ b/app/lib/ostatus/atom_serializer.rb
@@ -354,7 +354,7 @@ class OStatus::AtomSerializer
     append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
     append_element(entry, 'content', Formatter.instance.format(status).to_str || '.', type: 'html', 'xml:lang': status.language)
 
-    status.mentions.sort_by(&:id).each do |mentioned|
+    status.active_mentions.sort_by(&:id).each do |mentioned|
       append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))
     end
 
diff --git a/app/models/account_conversation.rb b/app/models/account_conversation.rb
index a7205ec1a..c12c8d233 100644
--- a/app/models/account_conversation.rb
+++ b/app/models/account_conversation.rb
@@ -85,7 +85,7 @@ class AccountConversation < ApplicationRecord
     private
 
     def participants_from_status(recipient, status)
-      ((status.mentions.pluck(:account_id) + [status.account_id]).uniq - [recipient.id]).sort
+      ((status.active_mentions.pluck(:account_id) + [status.account_id]).uniq - [recipient.id]).sort
     end
   end
 
diff --git a/app/models/mention.rb b/app/models/mention.rb
index 8ab886b18..d01a88e32 100644
--- a/app/models/mention.rb
+++ b/app/models/mention.rb
@@ -8,6 +8,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  account_id :bigint(8)
+#  silent     :boolean          default(FALSE), not null
 #
 
 class Mention < ApplicationRecord
@@ -18,10 +19,17 @@ class Mention < ApplicationRecord
 
   validates :account, uniqueness: { scope: :status }
 
+  scope :active, -> { where(silent: false) }
+  scope :silent, -> { where(silent: true) }
+
   delegate(
     :username,
     :acct,
     to: :account,
     prefix: true
   )
+
+  def active?
+    !silent?
+  end
 end
diff --git a/app/models/notification.rb b/app/models/notification.rb
index b9bec0808..78b180301 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -24,7 +24,7 @@ class Notification < ApplicationRecord
     favourite:      'Favourite',
   }.freeze
 
-  STATUS_INCLUDES = [:account, :application, :stream_entry, :media_attachments, :tags, mentions: :account, reblog: [:stream_entry, :account, :application, :media_attachments, :tags, mentions: :account]].freeze
+  STATUS_INCLUDES = [:account, :application, :media_attachments, :tags, active_mentions: :account, reblog: [:account, :application, :media_attachments, :tags, active_mentions: :account]].freeze
 
   belongs_to :account, optional: true
   belongs_to :from_account, class_name: 'Account', optional: true
diff --git a/app/models/status.rb b/app/models/status.rb
index f61bd0fee..b18cb56b2 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -37,7 +37,7 @@ class Status < ApplicationRecord
 
   update_index('statuses#status', :proper) if Chewy.enabled?
 
-  enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility
+  enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility
 
   belongs_to :application, class_name: 'Doorkeeper::Application', optional: true
 
@@ -51,7 +51,8 @@ class Status < ApplicationRecord
   has_many :favourites, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
-  has_many :mentions, dependent: :destroy
+  has_many :mentions, dependent: :destroy, inverse_of: :status
+  has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
   has_many :media_attachments, dependent: :nullify
 
   has_and_belongs_to_many :tags
@@ -89,7 +90,7 @@ class Status < ApplicationRecord
                    :status_stat,
                    :tags,
                    :stream_entry,
-                   mentions: :account,
+                   active_mentions: :account,
                    reblog: [
                      :account,
                      :application,
@@ -98,7 +99,7 @@ class Status < ApplicationRecord
                      :media_attachments,
                      :conversation,
                      :status_stat,
-                     mentions: :account,
+                     active_mentions: :account,
                    ],
                    thread: :account
 
@@ -171,7 +172,11 @@ class Status < ApplicationRecord
   end
 
   def hidden?
-    private_visibility? || direct_visibility?
+    private_visibility? || direct_visibility? || limited_visibility?
+  end
+
+  def distributable?
+    public_visibility? || unlisted_visibility?
   end
 
   def with_media?
diff --git a/app/models/stream_entry.rb b/app/models/stream_entry.rb
index a2f273281..1a9afc5c7 100644
--- a/app/models/stream_entry.rb
+++ b/app/models/stream_entry.rb
@@ -48,7 +48,7 @@ class StreamEntry < ApplicationRecord
   end
 
   def mentions
-    orphaned? ? [] : status.mentions.map(&:account)
+    orphaned? ? [] : status.active_mentions.map(&:account)
   end
 
   private
diff --git a/app/policies/status_policy.rb b/app/policies/status_policy.rb
index 6addc8a8a..64a5111fc 100644
--- a/app/policies/status_policy.rb
+++ b/app/policies/status_policy.rb
@@ -12,7 +12,7 @@ class StatusPolicy < ApplicationPolicy
   end
 
   def show?
-    if direct?
+    if requires_mention?
       owned? || mention_exists?
     elsif private?
       owned? || following_author? || mention_exists?
@@ -22,7 +22,7 @@ class StatusPolicy < ApplicationPolicy
   end
 
   def reblog?
-    !direct? && (!private? || owned?) && show? && !blocking_author?
+    !requires_mention? && (!private? || owned?) && show? && !blocking_author?
   end
 
   def favourite?
@@ -41,8 +41,8 @@ class StatusPolicy < ApplicationPolicy
 
   private
 
-  def direct?
-    record.direct_visibility?
+  def requires_mention?
+    record.direct_visibility? || record.limited_visibility?
   end
 
   def owned?
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 82b7ffe95..c9d23e25f 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -68,7 +68,7 @@ class ActivityPub::NoteSerializer < ActiveModel::Serializer
   end
 
   def virtual_tags
-    object.mentions.to_a.sort_by(&:id) + object.tags + object.emojis
+    object.active_mentions.to_a.sort_by(&:id) + object.tags + object.emojis
   end
 
   def atom_uri
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index 61423f961..1f2f46b7e 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -36,6 +36,17 @@ class REST::StatusSerializer < ActiveModel::Serializer
     !current_user.nil?
   end
 
+  def visibility
+    # This visibility is masked behind "private"
+    # to avoid API changes because there are no
+    # UX differences
+    if object.limited_visibility?
+      'private'
+    else
+      object.visibility
+    end
+  end
+
   def uri
     OStatus::TagManager.instance.uri_for(object)
   end
@@ -88,7 +99,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   end
 
   def ordered_mentions
-    object.mentions.to_a.sort_by(&:id)
+    object.active_mentions.to_a.sort_by(&:id)
   end
 
   class ApplicationSerializer < ActiveModel::Serializer
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 2fcb3cc66..b8ab58938 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -12,7 +12,7 @@ class BatchedRemoveStatusService < BaseService
   def call(statuses)
     statuses = Status.where(id: statuses.map(&:id)).includes(:account, :stream_entry).flat_map { |status| [status] + status.reblogs.includes(:account, :stream_entry).to_a }
 
-    @mentions = statuses.map { |s| [s.id, s.mentions.includes(:account).to_a] }.to_h
+    @mentions = statuses.map { |s| [s.id, s.active_mentions.includes(:account).to_a] }.to_h
     @tags     = statuses.map { |s| [s.id, s.tags.pluck(:name)] }.to_h
 
     @stream_entry_batches  = []
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 5ddddf3a9..7f2a91775 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -10,6 +10,8 @@ class FanOutOnWriteService < BaseService
 
     if status.direct_visibility?
       deliver_to_own_conversation(status)
+    elsif status.limited_visibility?
+      deliver_to_mentioned_followers(status)
     else
       deliver_to_self(status) if status.account.local?
       deliver_to_followers(status)
@@ -53,6 +55,16 @@ class FanOutOnWriteService < BaseService
     end
   end
 
+  def deliver_to_mentioned_followers(status)
+    Rails.logger.debug "Delivering status #{status.id} to limited followers"
+
+    status.mentions.includes(:account).each do |mention|
+      mentioned_account = mention.account
+      next if !mentioned_account.local? || !mentioned_account.following?(status.account) || FeedManager.instance.filter?(:home, status, mention.account_id)
+      FeedManager.instance.push_to_home(mentioned_account, status)
+    end
+  end
+
   def render_anonymous_payload(status)
     @payload = InlineRenderer.render(status, nil, :status)
     @payload = Oj.dump(event: :update, payload: @payload)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 1ee645e6d..11d28e783 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -8,7 +8,7 @@ class RemoveStatusService < BaseService
     @status       = status
     @account      = status.account
     @tags         = status.tags.pluck(:name).to_a
-    @mentions     = status.mentions.includes(:account).to_a
+    @mentions     = status.active_mentions.includes(:account).to_a
     @reblogs      = status.reblogs.to_a
     @stream_entry = status.stream_entry
     @options      = options
diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
index 0b204d437..6e6d0eda8 100644
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ b/app/views/stream_entries/_detailed_status.html.haml
@@ -51,7 +51,7 @@
     - if status.direct_visibility?
       %span.detailed-status__link<
         = fa_icon('envelope')
-    - elsif status.private_visibility?
+    - elsif status.private_visibility? || status.limited_visibility?
       %span.detailed-status__link<
         = fa_icon('lock')
     - else
diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb
index c2bfd4f2f..17c1ef7ff 100644
--- a/app/workers/activitypub/distribution_worker.rb
+++ b/app/workers/activitypub/distribution_worker.rb
@@ -23,7 +23,7 @@ class ActivityPub::DistributionWorker
   private
 
   def skip_distribution?
-    @status.direct_visibility?
+    @status.direct_visibility? || @status.limited_visibility?
   end
 
   def relayable?
diff --git a/app/workers/activitypub/reply_distribution_worker.rb b/app/workers/activitypub/reply_distribution_worker.rb
index fe99fc05f..c0ed3a1f4 100644
--- a/app/workers/activitypub/reply_distribution_worker.rb
+++ b/app/workers/activitypub/reply_distribution_worker.rb
@@ -9,7 +9,7 @@ class ActivityPub::ReplyDistributionWorker
     @status  = Status.find(status_id)
     @account = @status.thread&.account
 
-    return if @account.nil? || skip_distribution?
+    return unless @account.present? && @status.distributable?
 
     ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
       [signed_payload, @status.account_id, inbox_url]
@@ -20,10 +20,6 @@ class ActivityPub::ReplyDistributionWorker
 
   private
 
-  def skip_distribution?
-    @status.private_visibility? || @status.direct_visibility?
-  end
-
   def inboxes
     @inboxes ||= @account.followers.inboxes
   end
diff --git a/db/migrate/20181010141500_add_silent_to_mentions.rb b/db/migrate/20181010141500_add_silent_to_mentions.rb
new file mode 100644
index 000000000..dbb4fba26
--- /dev/null
+++ b/db/migrate/20181010141500_add_silent_to_mentions.rb
@@ -0,0 +1,23 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddSilentToMentions < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      add_column_with_default(
+        :mentions,
+        :silent,
+        :boolean,
+        allow_null: false,
+        default: false
+      )
+    end
+  end
+
+  def down
+    remove_column :mentions, :silent
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index bf6ab4e68..f79f26f16 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2018_10_07_025445) do
+ActiveRecord::Schema.define(version: 2018_10_10_141500) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -301,6 +301,7 @@ ActiveRecord::Schema.define(version: 2018_10_07_025445) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.bigint "account_id"
+    t.boolean "silent", default: false, null: false
     t.index ["account_id", "status_id"], name: "index_mentions_on_account_id_and_status_id", unique: true
     t.index ["status_id"], name: "index_mentions_on_status_id"
   end
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index 62b9db8c2..cd20b7c7c 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -105,6 +105,31 @@ RSpec.describe ActivityPub::Activity::Create do
       end
     end
 
+    context 'limited' do
+      let(:recipient) { Fabricate(:account) }
+
+      let(:object_json) do
+        {
+          id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+          type: 'Note',
+          content: 'Lorem ipsum',
+          to: ActivityPub::TagManager.instance.uri_for(recipient),
+        }
+      end
+
+      it 'creates status' do
+        status = sender.statuses.first
+
+        expect(status).to_not be_nil
+        expect(status.visibility).to eq 'limited'
+      end
+
+      it 'creates silent mention' do
+        status = sender.statuses.first
+        expect(status.mentions.first).to be_silent
+      end
+    end
+
     context 'direct' do
       let(:recipient) { Fabricate(:account) }
 
@@ -114,6 +139,10 @@ RSpec.describe ActivityPub::Activity::Create do
           type: 'Note',
           content: 'Lorem ipsum',
           to: ActivityPub::TagManager.instance.uri_for(recipient),
+          tag: {
+            type: 'Mention',
+            href: ActivityPub::TagManager.instance.uri_for(recipient),
+          },
         }
       end
 
-- 
cgit 


From fd5285658f477c5b6a9c7af0935b5c889729b2e0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 20 Oct 2018 08:02:44 +0200
Subject: Add option to block reports from domain (#8830)

---
 app/controllers/admin/domain_blocks_controller.rb   |  2 +-
 app/javascript/packs/admin.js                       | 21 ++++++++-------------
 app/lib/activitypub/activity/flag.rb                |  8 ++++++++
 app/models/domain_block.rb                          | 13 +++++++------
 .../admin/domain_blocks/_domain_block.html.haml     |  5 ++++-
 app/views/admin/domain_blocks/index.html.haml       |  1 +
 app/views/admin/domain_blocks/new.html.haml         |  3 +++
 .../_email_domain_block.html.haml                   |  2 +-
 app/views/admin/instances/_instance.html.haml       |  2 +-
 config/locales/en.yml                               |  2 ++
 ...017170937_add_reject_reports_to_domain_blocks.rb | 17 +++++++++++++++++
 db/schema.rb                                        |  1 +
 12 files changed, 54 insertions(+), 23 deletions(-)
 create mode 100644 db/migrate/20181017170937_add_reject_reports_to_domain_blocks.rb

(limited to 'app/lib')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 64de2cbf0..90c70275a 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -46,7 +46,7 @@ module Admin
     end
 
     def resource_params
-      params.require(:domain_block).permit(:domain, :severity, :reject_media, :retroactive)
+      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :retroactive)
     end
 
     def retroactive_unblock?
diff --git a/app/javascript/packs/admin.js b/app/javascript/packs/admin.js
index ce5f70ee7..f0c0ee0b7 100644
--- a/app/javascript/packs/admin.js
+++ b/app/javascript/packs/admin.js
@@ -1,17 +1,5 @@
 import { delegate } from 'rails-ujs';
 
-function handleDeleteStatus(event) {
-  const [data] = event.detail;
-  const element = document.querySelector(`[data-id="${data.id}"]`);
-  if (element) {
-    element.parentNode.removeChild(element);
-  }
-}
-
-[].forEach.call(document.querySelectorAll('.trash-button'), (content) => {
-  content.addEventListener('ajax:success', handleDeleteStatus);
-});
-
 const batchCheckboxClassName = '.batch-checkbox input[type="checkbox"]';
 
 delegate(document, '#batch_checkbox_all', 'change', ({ target }) => {
@@ -22,6 +10,7 @@ delegate(document, '#batch_checkbox_all', 'change', ({ target }) => {
 
 delegate(document, batchCheckboxClassName, 'change', () => {
   const checkAllElement = document.querySelector('#batch_checkbox_all');
+
   if (checkAllElement) {
     checkAllElement.checked = [].every.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked);
     checkAllElement.indeterminate = !checkAllElement.checked && [].some.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked);
@@ -41,8 +30,14 @@ delegate(document, '.media-spoiler-hide-button', 'click', () => {
 });
 
 delegate(document, '#domain_block_severity', 'change', ({ target }) => {
-  const rejectMediaDiv = document.querySelector('.input.with_label.domain_block_reject_media');
+  const rejectMediaDiv   = document.querySelector('.input.with_label.domain_block_reject_media');
+  const rejectReportsDiv = document.querySelector('.input.with_label.domain_block_reject_reports');
+
   if (rejectMediaDiv) {
     rejectMediaDiv.style.display = (target.value === 'suspend') ? 'none' : 'block';
   }
+
+  if (rejectReportsDiv) {
+    rejectReportsDiv.style.display = (target.value === 'suspend') ? 'none' : 'block';
+  }
 });
diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb
index 36d3c5730..92e59bb81 100644
--- a/app/lib/activitypub/activity/flag.rb
+++ b/app/lib/activitypub/activity/flag.rb
@@ -2,6 +2,8 @@
 
 class ActivityPub::Activity::Flag < ActivityPub::Activity
   def perform
+    return if skip_reports?
+
     target_accounts            = object_uris.map { |uri| account_from_uri(uri) }.compact.select(&:local?)
     target_statuses_by_account = object_uris.map { |uri| status_from_uri(uri) }.compact.select(&:local?).group_by(&:account_id)
 
@@ -19,6 +21,12 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
     end
   end
 
+  private
+
+  def skip_reports?
+    DomainBlock.find_by(domain: @account.domain)&.reject_reports?
+  end
+
   def object_uris
     @object_uris ||= Array(@object.is_a?(Array) ? @object.map { |item| value_or_id(item) } : value_or_id(@object))
   end
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 93658793b..b828a9d71 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -3,12 +3,13 @@
 #
 # Table name: domain_blocks
 #
-#  id           :bigint(8)        not null, primary key
-#  domain       :string           default(""), not null
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
-#  severity     :integer          default("silence")
-#  reject_media :boolean          default(FALSE), not null
+#  id             :bigint(8)        not null, primary key
+#  domain         :string           default(""), not null
+#  created_at     :datetime         not null
+#  updated_at     :datetime         not null
+#  severity       :integer          default("silence")
+#  reject_media   :boolean          default(FALSE), not null
+#  reject_reports :boolean          default(FALSE), not null
 #
 
 class DomainBlock < ApplicationRecord
diff --git a/app/views/admin/domain_blocks/_domain_block.html.haml b/app/views/admin/domain_blocks/_domain_block.html.haml
index 17a3de81c..7bfea3574 100644
--- a/app/views/admin/domain_blocks/_domain_block.html.haml
+++ b/app/views/admin/domain_blocks/_domain_block.html.haml
@@ -1,10 +1,13 @@
 %tr
-  %td.domain
+  %td
     %samp= domain_block.domain
   %td.severity
     = t("admin.domain_blocks.severities.#{domain_block.severity}")
   %td.reject_media
     - if domain_block.reject_media? || domain_block.suspend?
       %i.fa.fa-check
+  %td.reject_reports
+    - if domain_block.reject_reports? || domain_block.suspend?
+      %i.fa.fa-check
   %td
     = table_link_to 'undo', t('admin.domain_blocks.undo'), admin_domain_block_path(domain_block)
diff --git a/app/views/admin/domain_blocks/index.html.haml b/app/views/admin/domain_blocks/index.html.haml
index 42b8ca53f..4c5221c42 100644
--- a/app/views/admin/domain_blocks/index.html.haml
+++ b/app/views/admin/domain_blocks/index.html.haml
@@ -8,6 +8,7 @@
         %th= t('admin.domain_blocks.domain')
         %th= t('admin.domain_blocks.severity')
         %th= t('admin.domain_blocks.reject_media')
+        %th= t('admin.domain_blocks.reject_reports')
         %th
     %tbody
       = render @domain_blocks
diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml
index f0eb217eb..055d2fbd7 100644
--- a/app/views/admin/domain_blocks/new.html.haml
+++ b/app/views/admin/domain_blocks/new.html.haml
@@ -17,5 +17,8 @@
   .fields-group
     = f.input :reject_media, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_media'), hint: I18n.t('admin.domain_blocks.reject_media_hint')
 
+  .fields-group
+    = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
+
   .actions
     = f.button :button, t('.create'), type: :submit
diff --git a/app/views/admin/email_domain_blocks/_email_domain_block.html.haml b/app/views/admin/email_domain_blocks/_email_domain_block.html.haml
index 61cff9395..bf66c9001 100644
--- a/app/views/admin/email_domain_blocks/_email_domain_block.html.haml
+++ b/app/views/admin/email_domain_blocks/_email_domain_block.html.haml
@@ -1,5 +1,5 @@
 %tr
-  %td.domain
+  %td
     %samp= email_domain_block.domain
   %td
     = table_link_to 'trash', t('admin.email_domain_blocks.delete'), admin_email_domain_block_path(email_domain_block), method: :delete
diff --git a/app/views/admin/instances/_instance.html.haml b/app/views/admin/instances/_instance.html.haml
index 6efbbbe60..e36ebae47 100644
--- a/app/views/admin/instances/_instance.html.haml
+++ b/app/views/admin/instances/_instance.html.haml
@@ -1,5 +1,5 @@
 %tr
-  %td.domain
+  %td
     = link_to instance.domain, admin_accounts_path(by_domain: instance.domain)
   %td.count
     = instance.accounts_count
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 0360e719e..a2859aa5d 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -263,6 +263,8 @@ en:
         title: New domain block
       reject_media: Reject media files
       reject_media_hint: Removes locally stored media files and refuses to download any in the future. Irrelevant for suspensions
+      reject_reports: Reject reports
+      reject_reports_hint: Ignore all reports coming from this domain. Irrelevant for suspensions
       severities:
         noop: None
         silence: Silence
diff --git a/db/migrate/20181017170937_add_reject_reports_to_domain_blocks.rb b/db/migrate/20181017170937_add_reject_reports_to_domain_blocks.rb
new file mode 100644
index 000000000..f05d50fcd
--- /dev/null
+++ b/db/migrate/20181017170937_add_reject_reports_to_domain_blocks.rb
@@ -0,0 +1,17 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddRejectReportsToDomainBlocks < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      add_column_with_default :domain_blocks, :reject_reports, :boolean, default: false, allow_null: false
+    end
+  end
+
+  def down
+    remove_column :domain_blocks, :reject_reports
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 046975ac9..8facfa259 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -186,6 +186,7 @@ ActiveRecord::Schema.define(version: 2018_10_18_205649) do
     t.datetime "updated_at", null: false
     t.integer "severity", default: 0
     t.boolean "reject_media", default: false, null: false
+    t.boolean "reject_reports", default: false, null: false
     t.index ["domain"], name: "index_domain_blocks_on_domain", unique: true
   end
 
-- 
cgit